2024-04-12T08:24:14.953 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-12T08:24:14.953 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --name teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi --verbose --archive /home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395 --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} -- /home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395/orig.config.yaml 2024-04-12T08:24:15.014 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-12T08:24:15.108 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7653395' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: dea717a37c7849ca8459c02fb965020aedcc0a9a owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4203 sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 26/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_dea717a37c7849ca8459c02fb965020aedcc0a9a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: dea717a37c7849ca8459c02fb965020aedcc0a9a targets: smithi090.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGqN0HyNQTnN1DzR4tPZQnLo35xJt4BjNPTkJg2QGWKxVXg8amz0AqCYugiw0t6WgiLxfK5Hl6VvRzOV09zbcVg= smithi114.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPSxYvJSi9IeAMhdznrwLqJLH3yDDYmhIQ0mElMNI+MMohDyPYFjKRuNV8InIpNIW5G9dgzsbNhnH+ElEFebY4A= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-12_02:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-12T08:24:15.108 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_dea717a37c7849ca8459c02fb965020aedcc0a9a/qa; will attempt to use it 2024-04-12T08:24:15.109 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_dea717a37c7849ca8459c02fb965020aedcc0a9a/qa/tasks 2024-04-12T08:24:15.109 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-12T08:24:15.110 INFO:teuthology.task.internal:Checking packages... 2024-04-12T08:24:15.133 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b277feb2c7beb4597d8ecc39d58d5c36b81f613a' 2024-04-12T08:24:15.133 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T08:24:15.133 INFO:teuthology.packaging:ref: None 2024-04-12T08:24:15.133 INFO:teuthology.packaging:tag: None 2024-04-12T08:24:15.133 INFO:teuthology.packaging:branch: squid 2024-04-12T08:24:15.133 INFO:teuthology.packaging:sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:24:15.133 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-12T08:24:15.434 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2053.gb277feb2 2024-04-12T08:24:15.436 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-12T08:24:15.442 INFO:teuthology.task.internal:no buildpackages task found 2024-04-12T08:24:15.442 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-12T08:24:15.474 INFO:teuthology.task.internal:Saving configuration 2024-04-12T08:24:15.496 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-12T08:24:15.501 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T08:24:15.527 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi090.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 08:16:13.323032', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGqN0HyNQTnN1DzR4tPZQnLo35xJt4BjNPTkJg2QGWKxVXg8amz0AqCYugiw0t6WgiLxfK5Hl6VvRzOV09zbcVg='} 2024-04-12T08:24:15.548 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi114.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 08:16:13.321447', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPSxYvJSi9IeAMhdznrwLqJLH3yDDYmhIQ0mElMNI+MMohDyPYFjKRuNV8InIpNIW5G9dgzsbNhnH+ElEFebY4A='} 2024-04-12T08:24:15.548 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-12T08:24:15.554 INFO:teuthology.task.internal:roles: ubuntu@smithi090.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-12T08:24:15.554 INFO:teuthology.task.internal:roles: ubuntu@smithi114.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-12T08:24:15.555 INFO:teuthology.run_tasks:Running task console_log... 2024-04-12T08:24:15.647 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fb0359140d0>, signals=[15]) 2024-04-12T08:24:15.647 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-12T08:24:15.655 INFO:teuthology.task.internal:Opening connections... 2024-04-12T08:24:15.655 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi090.front.sepia.ceph.com 2024-04-12T08:24:15.657 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi090.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:24:15.746 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi114.front.sepia.ceph.com 2024-04-12T08:24:15.747 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:24:15.836 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-12T08:24:15.843 DEBUG:teuthology.orchestra.run.smithi090:> uname -m 2024-04-12T08:24:15.860 INFO:teuthology.orchestra.run.smithi090.stdout:x86_64 2024-04-12T08:24:15.860 DEBUG:teuthology.orchestra.run.smithi090:> cat /etc/os-release 2024-04-12T08:24:15.916 INFO:teuthology.orchestra.run.smithi090.stdout:NAME="CentOS Stream" 2024-04-12T08:24:15.916 INFO:teuthology.orchestra.run.smithi090.stdout:VERSION="9" 2024-04-12T08:24:15.916 INFO:teuthology.orchestra.run.smithi090.stdout:ID="centos" 2024-04-12T08:24:15.916 INFO:teuthology.orchestra.run.smithi090.stdout:ID_LIKE="rhel fedora" 2024-04-12T08:24:15.916 INFO:teuthology.orchestra.run.smithi090.stdout:VERSION_ID="9" 2024-04-12T08:24:15.916 INFO:teuthology.orchestra.run.smithi090.stdout:PLATFORM_ID="platform:el9" 2024-04-12T08:24:15.916 INFO:teuthology.orchestra.run.smithi090.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-12T08:24:15.916 INFO:teuthology.orchestra.run.smithi090.stdout:ANSI_COLOR="0;31" 2024-04-12T08:24:15.916 INFO:teuthology.orchestra.run.smithi090.stdout:LOGO="fedora-logo-icon" 2024-04-12T08:24:15.917 INFO:teuthology.orchestra.run.smithi090.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-12T08:24:15.917 INFO:teuthology.orchestra.run.smithi090.stdout:HOME_URL="https://centos.org/" 2024-04-12T08:24:15.917 INFO:teuthology.orchestra.run.smithi090.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-12T08:24:15.917 INFO:teuthology.orchestra.run.smithi090.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-12T08:24:15.917 INFO:teuthology.orchestra.run.smithi090.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-12T08:24:15.917 INFO:teuthology.lock.ops:Updating smithi090.front.sepia.ceph.com on lock server 2024-04-12T08:24:15.941 DEBUG:teuthology.orchestra.run.smithi114:> uname -m 2024-04-12T08:24:15.959 INFO:teuthology.orchestra.run.smithi114.stdout:x86_64 2024-04-12T08:24:15.959 DEBUG:teuthology.orchestra.run.smithi114:> cat /etc/os-release 2024-04-12T08:24:16.016 INFO:teuthology.orchestra.run.smithi114.stdout:NAME="CentOS Stream" 2024-04-12T08:24:16.016 INFO:teuthology.orchestra.run.smithi114.stdout:VERSION="9" 2024-04-12T08:24:16.016 INFO:teuthology.orchestra.run.smithi114.stdout:ID="centos" 2024-04-12T08:24:16.016 INFO:teuthology.orchestra.run.smithi114.stdout:ID_LIKE="rhel fedora" 2024-04-12T08:24:16.017 INFO:teuthology.orchestra.run.smithi114.stdout:VERSION_ID="9" 2024-04-12T08:24:16.017 INFO:teuthology.orchestra.run.smithi114.stdout:PLATFORM_ID="platform:el9" 2024-04-12T08:24:16.017 INFO:teuthology.orchestra.run.smithi114.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-12T08:24:16.017 INFO:teuthology.orchestra.run.smithi114.stdout:ANSI_COLOR="0;31" 2024-04-12T08:24:16.017 INFO:teuthology.orchestra.run.smithi114.stdout:LOGO="fedora-logo-icon" 2024-04-12T08:24:16.017 INFO:teuthology.orchestra.run.smithi114.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-12T08:24:16.017 INFO:teuthology.orchestra.run.smithi114.stdout:HOME_URL="https://centos.org/" 2024-04-12T08:24:16.017 INFO:teuthology.orchestra.run.smithi114.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-12T08:24:16.017 INFO:teuthology.orchestra.run.smithi114.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-12T08:24:16.017 INFO:teuthology.orchestra.run.smithi114.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-12T08:24:16.017 INFO:teuthology.lock.ops:Updating smithi114.front.sepia.ceph.com on lock server 2024-04-12T08:24:16.041 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-12T08:24:16.050 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-12T08:24:16.090 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-12T08:24:16.091 DEBUG:teuthology.orchestra.run.smithi090:> test '!' -e /home/ubuntu/cephtest 2024-04-12T08:24:16.093 DEBUG:teuthology.orchestra.run.smithi114:> test '!' -e /home/ubuntu/cephtest 2024-04-12T08:24:16.109 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-12T08:24:16.124 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-12T08:24:16.124 DEBUG:teuthology.orchestra.run.smithi090:> test -z $(ls -A /var/lib/ceph) 2024-04-12T08:24:16.150 DEBUG:teuthology.orchestra.run.smithi114:> test -z $(ls -A /var/lib/ceph) 2024-04-12T08:24:16.248 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-12T08:24:16.303 INFO:teuthology.run_tasks:Running task kernel... 2024-04-12T08:24:16.319 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-12T08:24:16.319 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-12T08:24:16.319 DEBUG:teuthology.orchestra.run.smithi090:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-12T08:24:16.320 DEBUG:teuthology.orchestra.run.smithi114:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-12T08:24:16.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T08:24:16.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T08:24:16.337 DEBUG:teuthology.orchestra.run.smithi114:> uname -r 2024-04-12T08:24:16.337 DEBUG:teuthology.orchestra.run.smithi090:> uname -r 2024-04-12T08:24:16.393 INFO:teuthology.orchestra.run.smithi114.stdout:5.14.0-361.el9.x86_64 2024-04-12T08:24:16.393 INFO:teuthology.task.kernel:Running kernel on smithi114: 5.14.0-361.el9.x86_64 2024-04-12T08:24:16.393 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum install -y kernel 2024-04-12T08:24:16.394 INFO:teuthology.orchestra.run.smithi090.stdout:5.14.0-361.el9.x86_64 2024-04-12T08:24:16.395 INFO:teuthology.task.kernel:Running kernel on smithi090: 5.14.0-361.el9.x86_64 2024-04-12T08:24:16.395 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum install -y kernel 2024-04-12T08:24:18.686 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS-9 - AppStream 13 MB/s | 8.1 MB 00:00 2024-04-12T08:24:18.880 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS-9 - AppStream 16 MB/s | 8.1 MB 00:00 2024-04-12T08:24:20.361 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS-9 - BaseOS 7.0 MB/s | 2.5 MB 00:00 2024-04-12T08:24:20.513 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS-9 - BaseOS 7.8 MB/s | 2.5 MB 00:00 2024-04-12T08:24:21.444 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.0 MB 00:00 2024-04-12T08:24:21.463 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.0 MB 00:00 2024-04-12T08:24:23.373 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS Stream 9 - AppStream 24 MB/s | 19 MB 00:00 2024-04-12T08:24:27.070 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - AppStream 4.2 MB/s | 19 MB 00:04 2024-04-12T08:24:28.382 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS Stream 9 - CRB 4.4 MB/s | 6.2 MB 00:01 2024-04-12T08:24:29.985 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS Stream 9 - Extras packages 36 kB/s | 16 kB 00:00 2024-04-12T08:24:31.435 INFO:teuthology.orchestra.run.smithi090.stdout:Extra Packages for Enterprise Linux 16 MB/s | 21 MB 00:01 2024-04-12T08:24:32.079 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - CRB 4.4 MB/s | 6.2 MB 00:01 2024-04-12T08:24:33.779 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - Extras packages 30 kB/s | 16 kB 00:00 2024-04-12T08:24:36.429 INFO:teuthology.orchestra.run.smithi090.stdout:lab-extras 38 kB/s | 1.7 kB 00:00 2024-04-12T08:24:38.670 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:24:38.670 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:24:38.670 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:24:38.670 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.670 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:24:38.670 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:24:38.671 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:24:38.672 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:24:38.673 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:24:38.674 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.675 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.676 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.677 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:24:38.678 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.678 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.678 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.678 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.678 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:24:38.678 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.678 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.678 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:24:38.678 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:24:39.078 INFO:teuthology.orchestra.run.smithi090.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-12T08:24:39.078 INFO:teuthology.orchestra.run.smithi090.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-12T08:24:39.208 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout: Package Architecture Version Repository Size 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout: kernel x86_64 5.14.0-435.el9 baseos 214 k 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout: kernel-core x86_64 5.14.0-435.el9 baseos 15 M 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout: kernel-modules x86_64 5.14.0-435.el9 baseos 34 M 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout: kernel-modules-core x86_64 5.14.0-435.el9 baseos 28 M 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:24:39.214 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:24:39.215 INFO:teuthology.orchestra.run.smithi090.stdout:Install 4 Packages 2024-04-12T08:24:39.215 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:24:39.215 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 77 M 2024-04-12T08:24:39.215 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 123 M 2024-04-12T08:24:39.215 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:24:39.497 INFO:teuthology.orchestra.run.smithi090.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 1.4 MB/s | 214 kB 00:00 2024-04-12T08:24:39.948 INFO:teuthology.orchestra.run.smithi090.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 25 MB/s | 15 MB 00:00 2024-04-12T08:24:40.473 INFO:teuthology.orchestra.run.smithi090.stdout:(3/4): kernel-modules-core-5.14.0-435.el9.x86_6 28 MB/s | 28 MB 00:00 2024-04-12T08:24:40.899 INFO:teuthology.orchestra.run.smithi090.stdout:(4/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 22 MB/s | 34 MB 00:01 2024-04-12T08:24:40.899 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:24:40.899 INFO:teuthology.orchestra.run.smithi090.stdout:Total 45 MB/s | 77 MB 00:01 2024-04-12T08:24:41.570 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:24:41.956 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:24:41.956 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:24:42.938 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:24:42.939 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:24:45.474 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:24:46.124 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-12T08:24:46.205 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T08:24:48.960 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T08:24:49.099 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T08:24:51.065 INFO:teuthology.orchestra.run.smithi114.stdout:Extra Packages for Enterprise Linux 1.2 MB/s | 21 MB 00:17 2024-04-12T08:24:53.789 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T08:24:53.827 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:24:56.370 INFO:teuthology.orchestra.run.smithi114.stdout:lab-extras 5.6 kB/s | 1.7 kB 00:00 2024-04-12T08:24:58.575 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:24:58.610 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:24:58.610 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.611 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:24:58.612 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:24:58.613 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.614 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:24:58.615 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.616 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.617 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.618 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.619 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:24:58.619 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.619 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.619 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.619 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:24:58.986 INFO:teuthology.orchestra.run.smithi114.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-12T08:24:58.986 INFO:teuthology.orchestra.run.smithi114.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-12T08:24:59.104 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:24:59.114 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout: Package Architecture Version Repository Size 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout: kernel x86_64 5.14.0-435.el9 baseos 214 k 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout:Installing dependencies: 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-core x86_64 5.14.0-435.el9 baseos 15 M 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-modules x86_64 5.14.0-435.el9 baseos 34 M 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-modules-core x86_64 5.14.0-435.el9 baseos 28 M 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout:Install 4 Packages 2024-04-12T08:24:59.115 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:24:59.116 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 77 M 2024-04-12T08:24:59.116 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 123 M 2024-04-12T08:24:59.116 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:24:59.330 INFO:teuthology.orchestra.run.smithi114.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 1.4 MB/s | 214 kB 00:00 2024-04-12T08:24:59.772 INFO:teuthology.orchestra.run.smithi114.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 25 MB/s | 15 MB 00:00 2024-04-12T08:25:00.223 INFO:teuthology.orchestra.run.smithi114.stdout:(3/4): kernel-modules-core-5.14.0-435.el9.x86_6 31 MB/s | 28 MB 00:00 2024-04-12T08:25:00.556 INFO:teuthology.orchestra.run.smithi114.stdout:(4/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 24 MB/s | 34 MB 00:01 2024-04-12T08:25:00.556 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:25:00.556 INFO:teuthology.orchestra.run.smithi114.stdout:Total 53 MB/s | 77 MB 00:01 2024-04-12T08:25:01.226 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:25:01.593 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:25:01.593 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:25:02.654 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:25:02.654 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:25:05.119 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:25:05.697 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-12T08:25:05.789 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T08:25:08.535 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T08:25:08.666 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T08:25:13.414 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T08:25:13.525 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:25:18.296 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:25:29.100 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:25:29.100 INFO:teuthology.orchestra.run.smithi090.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-12T08:25:29.100 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:25:29.146 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:25:29.619 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:25:29.619 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-12T08:25:29.619 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T08:25:29.619 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T08:25:30.111 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:25:30.111 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:25:30.111 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:25:30.111 INFO:teuthology.orchestra.run.smithi090.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-12T08:25:30.111 INFO:teuthology.orchestra.run.smithi090.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-12T08:25:30.111 INFO:teuthology.orchestra.run.smithi090.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-12T08:25:30.112 INFO:teuthology.orchestra.run.smithi090.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-12T08:25:30.112 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:25:30.112 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:25:30.491 DEBUG:teuthology.orchestra.run.smithi090:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T08:25:30.620 INFO:teuthology.orchestra.run.smithi090.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T08:25:30.621 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T08:25:30.621 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-12T08:25:30.621 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-12T08:25:30.621 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-12T08:25:30.621 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-12T08:25:30.621 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi090.front.sepia.ceph.com, path=None, version=distro) 2024-04-12T08:25:30.621 DEBUG:teuthology.orchestra.run.smithi090:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T08:25:30.703 INFO:teuthology.orchestra.run.smithi090.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T08:25:30.703 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T08:25:30.703 DEBUG:teuthology.orchestra.run.smithi090:> sudo rpm -qi grub2-tools 2024-04-12T08:25:30.736 INFO:teuthology.orchestra.run.smithi090.stdout:Name : grub2-tools 2024-04-12T08:25:30.737 INFO:teuthology.orchestra.run.smithi090.stdout:Epoch : 1 2024-04-12T08:25:30.737 INFO:teuthology.orchestra.run.smithi090.stdout:Version : 2.06 2024-04-12T08:25:30.737 INFO:teuthology.orchestra.run.smithi090.stdout:Release : 68.el9 2024-04-12T08:25:30.737 INFO:teuthology.orchestra.run.smithi090.stdout:Architecture: x86_64 2024-04-12T08:25:30.737 INFO:teuthology.orchestra.run.smithi090.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-12T08:25:30.737 INFO:teuthology.orchestra.run.smithi090.stdout:Group : Unspecified 2024-04-12T08:25:30.738 INFO:teuthology.orchestra.run.smithi090.stdout:Size : 8264254 2024-04-12T08:25:30.738 INFO:teuthology.orchestra.run.smithi090.stdout:License : GPLv3+ 2024-04-12T08:25:30.738 INFO:teuthology.orchestra.run.smithi090.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-12T08:25:30.738 INFO:teuthology.orchestra.run.smithi090.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-12T08:25:30.738 INFO:teuthology.orchestra.run.smithi090.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-12T08:25:30.738 INFO:teuthology.orchestra.run.smithi090.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-12T08:25:30.738 INFO:teuthology.orchestra.run.smithi090.stdout:Packager : builder@centos.org 2024-04-12T08:25:30.738 INFO:teuthology.orchestra.run.smithi090.stdout:Vendor : CentOS 2024-04-12T08:25:30.739 INFO:teuthology.orchestra.run.smithi090.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-12T08:25:30.739 INFO:teuthology.orchestra.run.smithi090.stdout:Summary : Support tools for GRUB. 2024-04-12T08:25:30.739 INFO:teuthology.orchestra.run.smithi090.stdout:Description : 2024-04-12T08:25:30.739 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:25:30.739 INFO:teuthology.orchestra.run.smithi090.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-12T08:25:30.740 INFO:teuthology.orchestra.run.smithi090.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-12T08:25:30.740 INFO:teuthology.orchestra.run.smithi090.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-12T08:25:30.740 INFO:teuthology.orchestra.run.smithi090.stdout:hardware devices. 2024-04-12T08:25:30.740 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:25:30.740 INFO:teuthology.orchestra.run.smithi090.stdout:This subpackage provides tools for support of all platforms. 2024-04-12T08:25:30.741 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-12T08:25:30.741 INFO:teuthology.task.kernel:Updating grub on smithi090 to boot 5.14.0-435.el9.x86_64 2024-04-12T08:25:30.742 DEBUG:teuthology.orchestra.run.smithi090:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-12T08:25:31.395 INFO:teuthology.orchestra.run.smithi090.stderr:Generating grub configuration file ... 2024-04-12T08:25:33.247 INFO:teuthology.orchestra.run.smithi090.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-12T08:25:33.271 INFO:teuthology.orchestra.run.smithi090.stderr:done 2024-04-12T08:25:33.273 DEBUG:teuthology.orchestra.run.smithi090:> mktemp 2024-04-12T08:25:33.291 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/tmp.gqBFllY6tP 2024-04-12T08:25:33.292 DEBUG:teuthology.orchestra.run.smithi090:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.gqBFllY6tP 2024-04-12T08:25:33.360 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod 0666 /tmp/tmp.gqBFllY6tP 2024-04-12T08:25:33.515 DEBUG:teuthology.orchestra.remote:smithi090:/tmp/tmp.gqBFllY6tP is 6KB 2024-04-12T08:25:33.564 DEBUG:teuthology.orchestra.run.smithi090:> rm -fr /tmp/tmp.gqBFllY6tP 2024-04-12T08:25:33.578 DEBUG:teuthology.orchestra.run.smithi090:> sudo /bin/ls /boot/loader/entries || true 2024-04-12T08:25:33.646 INFO:teuthology.orchestra.run.smithi090.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-12T08:25:33.646 INFO:teuthology.orchestra.run.smithi090.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-12T08:25:33.646 INFO:teuthology.orchestra.run.smithi090.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-12T08:25:33.646 INFO:teuthology.orchestra.run.smithi090.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-12T08:25:33.648 DEBUG:teuthology.orchestra.run.smithi090:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-12T08:25:33.804 DEBUG:teuthology.orchestra.run.smithi090:> sudo shutdown -r now 2024-04-12T08:25:47.901 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:25:47.902 INFO:teuthology.orchestra.run.smithi114.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-12T08:25:47.902 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:25:47.947 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:25:48.291 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:25:48.291 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-12T08:25:48.291 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T08:25:48.291 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T08:25:48.729 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T08:25:48.729 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:25:48.729 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:25:48.729 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-12T08:25:48.729 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-12T08:25:48.729 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-12T08:25:48.729 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-12T08:25:48.729 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:25:48.729 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:25:49.061 DEBUG:teuthology.orchestra.run.smithi114:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T08:25:49.163 INFO:teuthology.orchestra.run.smithi114.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T08:25:49.163 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T08:25:49.163 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-12T08:25:49.163 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-12T08:25:49.164 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-12T08:25:49.164 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-12T08:25:49.164 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi114.front.sepia.ceph.com, path=None, version=distro) 2024-04-12T08:25:49.164 DEBUG:teuthology.orchestra.run.smithi114:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T08:25:49.243 INFO:teuthology.orchestra.run.smithi114.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T08:25:49.244 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T08:25:49.244 DEBUG:teuthology.orchestra.run.smithi114:> sudo rpm -qi grub2-tools 2024-04-12T08:25:49.279 INFO:teuthology.orchestra.run.smithi114.stdout:Name : grub2-tools 2024-04-12T08:25:49.279 INFO:teuthology.orchestra.run.smithi114.stdout:Epoch : 1 2024-04-12T08:25:49.279 INFO:teuthology.orchestra.run.smithi114.stdout:Version : 2.06 2024-04-12T08:25:49.279 INFO:teuthology.orchestra.run.smithi114.stdout:Release : 68.el9 2024-04-12T08:25:49.279 INFO:teuthology.orchestra.run.smithi114.stdout:Architecture: x86_64 2024-04-12T08:25:49.279 INFO:teuthology.orchestra.run.smithi114.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:Group : Unspecified 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:Size : 8264254 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:License : GPLv3+ 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:Packager : builder@centos.org 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:Vendor : CentOS 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:Summary : Support tools for GRUB. 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:Description : 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:25:49.280 INFO:teuthology.orchestra.run.smithi114.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-12T08:25:49.281 INFO:teuthology.orchestra.run.smithi114.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-12T08:25:49.281 INFO:teuthology.orchestra.run.smithi114.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-12T08:25:49.281 INFO:teuthology.orchestra.run.smithi114.stdout:hardware devices. 2024-04-12T08:25:49.281 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:25:49.281 INFO:teuthology.orchestra.run.smithi114.stdout:This subpackage provides tools for support of all platforms. 2024-04-12T08:25:49.282 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-12T08:25:49.282 INFO:teuthology.task.kernel:Updating grub on smithi114 to boot 5.14.0-435.el9.x86_64 2024-04-12T08:25:49.282 DEBUG:teuthology.orchestra.run.smithi114:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-12T08:25:49.929 INFO:teuthology.orchestra.run.smithi114.stderr:Generating grub configuration file ... 2024-04-12T08:25:51.218 INFO:teuthology.orchestra.run.smithi114.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-12T08:25:51.253 INFO:teuthology.orchestra.run.smithi114.stderr:done 2024-04-12T08:25:51.258 DEBUG:teuthology.orchestra.run.smithi114:> mktemp 2024-04-12T08:25:51.273 INFO:teuthology.orchestra.run.smithi114.stdout:/tmp/tmp.O7pMJBm5eo 2024-04-12T08:25:51.273 DEBUG:teuthology.orchestra.run.smithi114:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.O7pMJBm5eo 2024-04-12T08:25:51.341 DEBUG:teuthology.orchestra.run.smithi114:> sudo chmod 0666 /tmp/tmp.O7pMJBm5eo 2024-04-12T08:25:51.478 DEBUG:teuthology.orchestra.remote:smithi114:/tmp/tmp.O7pMJBm5eo is 6KB 2024-04-12T08:25:51.526 DEBUG:teuthology.orchestra.run.smithi114:> rm -fr /tmp/tmp.O7pMJBm5eo 2024-04-12T08:25:51.540 DEBUG:teuthology.orchestra.run.smithi114:> sudo /bin/ls /boot/loader/entries || true 2024-04-12T08:25:51.605 INFO:teuthology.orchestra.run.smithi114.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-12T08:25:51.605 INFO:teuthology.orchestra.run.smithi114.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-12T08:25:51.605 INFO:teuthology.orchestra.run.smithi114.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-12T08:25:51.605 INFO:teuthology.orchestra.run.smithi114.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-12T08:25:51.606 DEBUG:teuthology.orchestra.run.smithi114:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-12T08:25:51.757 DEBUG:teuthology.orchestra.run.smithi114:> sudo shutdown -r now 2024-04-12T08:26:03.818 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-12T08:26:03.819 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi090.front.sepia.ceph.com' 2024-04-12T08:26:03.820 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi090.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:26:21.778 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-12T08:26:21.778 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-04-12T08:26:21.779 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:26:22.366 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.90 2024-04-12T08:26:31.374 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi090.front.sepia.ceph.com' 2024-04-12T08:26:31.375 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi090.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:26:34.437 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.90 2024-04-12T08:26:40.286 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.114 2024-04-12T08:26:46.444 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi090.front.sepia.ceph.com' 2024-04-12T08:26:46.444 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi090.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:26:49.290 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-04-12T08:26:49.294 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:26:49.502 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.90 2024-04-12T08:26:52.354 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.114 2024-04-12T08:27:04.366 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-04-12T08:27:04.367 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:27:04.503 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi090.front.sepia.ceph.com' 2024-04-12T08:27:04.504 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi090.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:27:22.787 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.114 2024-04-12T08:27:23.038 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.90 2024-04-12T08:27:37.802 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-04-12T08:27:37.803 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:27:38.224 DEBUG:teuthology.orchestra.run.smithi114:> true 2024-04-12T08:27:38.630 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-04-12T08:27:38.630 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-435.el9.x86_64"... 2024-04-12T08:27:38.630 DEBUG:teuthology.orchestra.run.smithi114:> uname -r 2024-04-12T08:27:38.644 INFO:teuthology.orchestra.run.smithi114.stdout:5.14.0-435.el9.x86_64 2024-04-12T08:27:38.645 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-12T08:27:38.645 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-12T08:27:38.645 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-12T08:27:39.645 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-12T08:27:39.646 DEBUG:teuthology.orchestra.run.smithi114:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-12T08:27:39.714 INFO:teuthology.orchestra.run.smithi114.stdout:ttyS1 2024-04-12T08:27:39.757 DEBUG:teuthology.parallel:result is None 2024-04-12T08:27:41.039 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi090.front.sepia.ceph.com' 2024-04-12T08:27:41.040 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi090.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:27:41.618 DEBUG:teuthology.orchestra.run.smithi090:> true 2024-04-12T08:27:42.195 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi090.front.sepia.ceph.com' 2024-04-12T08:27:42.196 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-435.el9.x86_64"... 2024-04-12T08:27:42.196 DEBUG:teuthology.orchestra.run.smithi090:> uname -r 2024-04-12T08:27:42.210 INFO:teuthology.orchestra.run.smithi090.stdout:5.14.0-435.el9.x86_64 2024-04-12T08:27:42.211 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-12T08:27:42.211 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-12T08:27:42.211 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-12T08:27:43.212 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-12T08:27:43.212 DEBUG:teuthology.orchestra.run.smithi090:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-12T08:27:43.733 INFO:teuthology.orchestra.run.smithi090.stdout:ttyS1 2024-04-12T08:27:43.772 DEBUG:teuthology.parallel:result is None 2024-04-12T08:27:43.772 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-12T08:27:43.779 INFO:teuthology.task.internal:Creating test directory... 2024-04-12T08:27:43.779 DEBUG:teuthology.orchestra.run.smithi090:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-12T08:27:43.782 DEBUG:teuthology.orchestra.run.smithi114:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-12T08:27:43.808 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-12T08:27:43.814 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-12T08:27:43.863 INFO:teuthology.task.internal:Creating archive directory... 2024-04-12T08:27:43.863 DEBUG:teuthology.orchestra.run.smithi090:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-12T08:27:43.866 DEBUG:teuthology.orchestra.run.smithi114:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-12T08:27:43.925 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-12T08:27:43.932 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-12T08:27:43.933 DEBUG:teuthology.orchestra.run.smithi090:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-12T08:27:43.936 DEBUG:teuthology.orchestra.run.smithi114:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-12T08:27:43.969 INFO:teuthology.orchestra.run.smithi114.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T08:27:43.983 INFO:teuthology.orchestra.run.smithi114.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T08:27:43.987 INFO:teuthology.orchestra.run.smithi090.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T08:27:44.007 INFO:teuthology.orchestra.run.smithi090.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T08:27:44.014 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-12T08:27:44.024 INFO:teuthology.task.internal:Configuring sudo... 2024-04-12T08:27:44.024 DEBUG:teuthology.orchestra.run.smithi090:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-12T08:27:44.027 DEBUG:teuthology.orchestra.run.smithi114:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-12T08:27:44.074 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-12T08:27:44.098 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-12T08:27:44.098 DEBUG:teuthology.orchestra.run.smithi090:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-12T08:27:44.101 DEBUG:teuthology.orchestra.run.smithi114:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-12T08:27:44.126 DEBUG:teuthology.orchestra.run.smithi090:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T08:27:44.208 DEBUG:teuthology.orchestra.run.smithi090:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T08:27:44.274 DEBUG:teuthology.orchestra.run.smithi090:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T08:27:44.321 DEBUG:teuthology.orchestra.run.smithi090:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T08:27:44.377 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:27:44.377 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-12T08:27:44.427 DEBUG:teuthology.orchestra.run.smithi114:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T08:27:44.471 DEBUG:teuthology.orchestra.run.smithi114:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T08:27:44.526 DEBUG:teuthology.orchestra.run.smithi114:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T08:27:44.609 DEBUG:teuthology.orchestra.run.smithi114:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T08:27:44.656 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:27:44.656 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-12T08:27:44.724 DEBUG:teuthology.orchestra.run.smithi090:> sudo service rsyslog restart 2024-04-12T08:27:44.727 DEBUG:teuthology.orchestra.run.smithi114:> sudo service rsyslog restart 2024-04-12T08:27:44.781 INFO:teuthology.orchestra.run.smithi090.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T08:27:44.817 INFO:teuthology.orchestra.run.smithi114.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T08:27:45.213 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-12T08:27:45.220 INFO:teuthology.task.internal:Starting timer... 2024-04-12T08:27:45.220 INFO:teuthology.run_tasks:Running task pcp... 2024-04-12T08:27:45.253 INFO:teuthology.run_tasks:Running task selinux... 2024-04-12T08:27:45.302 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-12T08:27:45.303 DEBUG:teuthology.orchestra.run.smithi090:> sudo service auditd rotate 2024-04-12T08:27:45.443 INFO:teuthology.orchestra.run.smithi090.stdout:Rotating logs: 2024-04-12T08:27:45.445 DEBUG:teuthology.orchestra.run.smithi114:> sudo service auditd rotate 2024-04-12T08:27:45.498 INFO:teuthology.orchestra.run.smithi114.stdout:Rotating logs: 2024-04-12T08:27:45.500 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-12T08:27:45.500 DEBUG:teuthology.orchestra.run.smithi090:> /usr/sbin/getenforce 2024-04-12T08:27:45.579 INFO:teuthology.orchestra.run.smithi090.stdout:Permissive 2024-04-12T08:27:45.580 DEBUG:teuthology.orchestra.run.smithi114:> /usr/sbin/getenforce 2024-04-12T08:27:45.601 INFO:teuthology.orchestra.run.smithi114.stdout:Permissive 2024-04-12T08:27:45.601 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi090.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi114.front.sepia.ceph.com': 'permissive'} 2024-04-12T08:27:45.602 DEBUG:teuthology.orchestra.run.smithi090:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-12T08:27:45.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T08:27:45.629 DEBUG:teuthology.orchestra.run.smithi114:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-12T08:27:45.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T08:27:45.670 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-12T08:27:45.670 DEBUG:teuthology.orchestra.run.smithi090:> sudo /usr/sbin/setenforce permissive 2024-04-12T08:27:45.712 DEBUG:teuthology.orchestra.run.smithi114:> sudo /usr/sbin/setenforce permissive 2024-04-12T08:27:45.754 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-12T08:27:45.764 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-12T08:27:45.795 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-12T08:27:45.797 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi090.front.sepia.ceph.com,smithi114.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-12T08:32:32.457 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi090.front.sepia.ceph.com'), Remote(name='ubuntu@smithi114.front.sepia.ceph.com')] 2024-04-12T08:32:32.458 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi090.front.sepia.ceph.com' 2024-04-12T08:32:32.459 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi090.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:32:32.554 DEBUG:teuthology.orchestra.run.smithi090:> true 2024-04-12T08:32:32.633 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi090.front.sepia.ceph.com' 2024-04-12T08:32:32.633 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-04-12T08:32:32.634 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T08:32:32.728 DEBUG:teuthology.orchestra.run.smithi114:> true 2024-04-12T08:32:32.814 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-04-12T08:32:32.815 INFO:teuthology.run_tasks:Running task clock... 2024-04-12T08:32:32.824 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-12T08:32:32.824 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-12T08:32:32.825 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T08:32:32.827 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-12T08:32:32.827 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T08:32:32.855 INFO:teuthology.orchestra.run.smithi090.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-12T08:32:32.872 INFO:teuthology.orchestra.run.smithi090.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-12T08:32:32.888 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-12T08:32:32.900 INFO:teuthology.orchestra.run.smithi090.stderr:sudo: ntpd: command not found 2024-04-12T08:32:32.905 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-12T08:32:32.914 INFO:teuthology.orchestra.run.smithi090.stdout:506 Cannot talk to daemon 2024-04-12T08:32:32.930 INFO:teuthology.orchestra.run.smithi090.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-12T08:32:32.935 INFO:teuthology.orchestra.run.smithi114.stderr:sudo: ntpd: command not found 2024-04-12T08:32:32.946 INFO:teuthology.orchestra.run.smithi090.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-12T08:32:32.949 INFO:teuthology.orchestra.run.smithi114.stdout:506 Cannot talk to daemon 2024-04-12T08:32:32.966 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-12T08:32:32.982 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-12T08:32:32.997 INFO:teuthology.orchestra.run.smithi090.stderr:bash: line 1: ntpq: command not found 2024-04-12T08:32:33.001 INFO:teuthology.orchestra.run.smithi090.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T08:32:33.001 INFO:teuthology.orchestra.run.smithi090.stdout:=============================================================================== 2024-04-12T08:32:33.002 INFO:teuthology.orchestra.run.smithi090.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T08:32:33.002 INFO:teuthology.orchestra.run.smithi090.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T08:32:33.002 INFO:teuthology.orchestra.run.smithi090.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T08:32:33.002 INFO:teuthology.orchestra.run.smithi090.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T08:32:33.034 INFO:teuthology.orchestra.run.smithi114.stderr:bash: line 1: ntpq: command not found 2024-04-12T08:32:33.037 INFO:teuthology.orchestra.run.smithi114.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T08:32:33.037 INFO:teuthology.orchestra.run.smithi114.stdout:=============================================================================== 2024-04-12T08:32:33.037 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T08:32:33.038 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T08:32:33.038 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T08:32:33.038 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T08:32:33.039 INFO:teuthology.run_tasks:Running task install... 2024-04-12T08:32:33.047 DEBUG:teuthology.task.install:project ceph 2024-04-12T08:32:33.047 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b277feb2c7beb4597d8ecc39d58d5c36b81f613a'}} 2024-04-12T08:32:33.047 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b277feb2c7beb4597d8ecc39d58d5c36b81f613a'} 2024-04-12T08:32:33.047 INFO:teuthology.task.install:Using flavor: default 2024-04-12T08:32:33.053 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-12T08:32:33.053 INFO:teuthology.task.install:extra packages: [] 2024-04-12T08:32:33.053 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b277feb2c7beb4597d8ecc39d58d5c36b81f613a', 'tag': None, 'wait_for_package': False} 2024-04-12T08:32:33.054 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T08:32:33.054 INFO:teuthology.packaging:ref: None 2024-04-12T08:32:33.054 INFO:teuthology.packaging:tag: None 2024-04-12T08:32:33.054 INFO:teuthology.packaging:branch: quincy 2024-04-12T08:32:33.054 INFO:teuthology.packaging:sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:32:33.054 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-12T08:32:33.056 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b277feb2c7beb4597d8ecc39d58d5c36b81f613a', 'tag': None, 'wait_for_package': False} 2024-04-12T08:32:33.056 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T08:32:33.056 INFO:teuthology.packaging:ref: None 2024-04-12T08:32:33.056 INFO:teuthology.packaging:tag: None 2024-04-12T08:32:33.056 INFO:teuthology.packaging:branch: quincy 2024-04-12T08:32:33.057 INFO:teuthology.packaging:sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:32:33.057 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-12T08:32:33.363 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/d3b5060a17449edfae9385cdf792afec109fca5d/centos/9/flavors/default/ 2024-04-12T08:32:33.363 INFO:teuthology.task.install.rpm:Package version is 17.2.7-827.gd3b5060a 2024-04-12T08:32:33.389 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/d3b5060a17449edfae9385cdf792afec109fca5d/centos/9/flavors/default/ 2024-04-12T08:32:33.389 INFO:teuthology.task.install.rpm:Package version is 17.2.7-827.gd3b5060a 2024-04-12T08:32:33.499 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/d3b5060a17449edfae9385cdf792afec109fca5d/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/d3b5060a17449edfae9385cdf792afec109fca5d/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/d3b5060a17449edfae9385cdf792afec109fca5d/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-12T08:32:33.499 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:32:33.499 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-12T08:32:33.528 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-12T08:32:33.529 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T08:32:33.529 INFO:teuthology.packaging:ref: None 2024-04-12T08:32:33.529 INFO:teuthology.packaging:tag: None 2024-04-12T08:32:33.529 INFO:teuthology.packaging:branch: quincy 2024-04-12T08:32:33.529 INFO:teuthology.packaging:sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:32:33.529 DEBUG:teuthology.orchestra.run.smithi090:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-12T08:32:33.559 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/d3b5060a17449edfae9385cdf792afec109fca5d/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/d3b5060a17449edfae9385cdf792afec109fca5d/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/d3b5060a17449edfae9385cdf792afec109fca5d/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-12T08:32:33.559 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:32:33.559 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-12T08:32:33.589 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-12T08:32:33.589 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T08:32:33.589 INFO:teuthology.packaging:ref: None 2024-04-12T08:32:33.589 INFO:teuthology.packaging:tag: None 2024-04-12T08:32:33.589 INFO:teuthology.packaging:branch: quincy 2024-04-12T08:32:33.589 INFO:teuthology.packaging:sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:32:33.589 DEBUG:teuthology.orchestra.run.smithi114:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-12T08:32:33.598 DEBUG:teuthology.orchestra.run.smithi090:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-12T08:32:33.658 DEBUG:teuthology.orchestra.run.smithi114:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-12T08:32:33.677 DEBUG:teuthology.orchestra.run.smithi090:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-12T08:32:33.739 DEBUG:teuthology.orchestra.run.smithi114:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-12T08:32:33.743 INFO:teuthology.orchestra.run.smithi090.stdout:check_obsoletes = 1 2024-04-12T08:32:33.745 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum clean all 2024-04-12T08:32:33.806 INFO:teuthology.orchestra.run.smithi114.stdout:check_obsoletes = 1 2024-04-12T08:32:33.807 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum clean all 2024-04-12T08:32:33.981 INFO:teuthology.orchestra.run.smithi090.stdout:74 files removed 2024-04-12T08:32:34.007 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-radosgw 2024-04-12T08:32:34.046 INFO:teuthology.orchestra.run.smithi114.stdout:74 files removed 2024-04-12T08:32:34.072 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph-radosgw 2024-04-12T08:32:34.837 INFO:teuthology.orchestra.run.smithi090.stdout:ceph packages for x86_64 146 kB/s | 77 kB 00:00 2024-04-12T08:32:34.856 INFO:teuthology.orchestra.run.smithi114.stdout:ceph packages for x86_64 159 kB/s | 77 kB 00:00 2024-04-12T08:32:35.337 INFO:teuthology.orchestra.run.smithi090.stdout:ceph noarch packages 20 kB/s | 9.2 kB 00:00 2024-04-12T08:32:35.607 INFO:teuthology.orchestra.run.smithi114.stdout:ceph noarch packages 13 kB/s | 9.2 kB 00:00 2024-04-12T08:32:36.315 INFO:teuthology.orchestra.run.smithi114.stdout:ceph source packages 2.7 kB/s | 1.8 kB 00:00 2024-04-12T08:32:36.893 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS-9 - AppStream 15 MB/s | 8.1 MB 00:00 2024-04-12T08:32:38.330 INFO:teuthology.orchestra.run.smithi090.stdout:ceph source packages 617 B/s | 1.8 kB 00:02 2024-04-12T08:32:38.551 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS-9 - BaseOS 7.5 MB/s | 2.5 MB 00:00 2024-04-12T08:32:38.933 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS-9 - AppStream 14 MB/s | 8.1 MB 00:00 2024-04-12T08:32:40.683 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS-9 - BaseOS 5.9 MB/s | 2.5 MB 00:00 2024-04-12T08:32:41.400 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - BaseOS 3.4 MB/s | 8.0 MB 00:02 2024-04-12T08:32:41.741 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.0 MB 00:00 2024-04-12T08:32:43.310 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - AppStream 24 MB/s | 19 MB 00:00 2024-04-12T08:32:48.927 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - CRB 3.0 MB/s | 6.2 MB 00:02 2024-04-12T08:32:50.970 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - Extras packages 19 kB/s | 16 kB 00:00 2024-04-12T08:32:56.637 INFO:teuthology.orchestra.run.smithi114.stdout:Extra Packages for Enterprise Linux 3.8 MB/s | 21 MB 00:05 2024-04-12T08:32:57.475 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS Stream 9 - AppStream 1.3 MB/s | 19 MB 00:14 2024-04-12T08:33:01.656 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS Stream 9 - CRB 9.2 MB/s | 6.2 MB 00:00 2024-04-12T08:33:01.787 INFO:teuthology.orchestra.run.smithi114.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-04-12T08:33:03.388 INFO:teuthology.orchestra.run.smithi090.stdout:CentOS Stream 9 - Extras packages 30 kB/s | 16 kB 00:00 2024-04-12T08:33:04.349 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:33:04.349 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:33:04.349 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:33:04.349 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.349 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:33:04.349 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.349 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:33:04.349 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.349 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.350 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:33:04.351 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.352 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.353 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:33:04.354 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.355 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:33:04.356 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.357 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:33:04.358 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.358 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.358 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.358 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:33:04.748 INFO:teuthology.orchestra.run.smithi090.stdout:Extra Packages for Enterprise Linux 17 MB/s | 21 MB 00:01 2024-04-12T08:33:04.861 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repository Size 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-radosgw x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 14 M 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout:Upgrading: 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout: librados2 x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 3.1 M 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout: librbd1 x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 2.9 M 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout:Installing dependencies: 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-base x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 5.0 M 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-common x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 22 M 2024-04-12T08:33:04.873 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-selinux x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 25 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: libcephfs2 x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 637 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: libradosstriper1 x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 458 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: librgw2 x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 4.9 M 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: python3-ceph-argparse x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 45 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: python3-ceph-common x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 117 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cephfs x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 156 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-12T08:33:04.874 INFO:teuthology.orchestra.run.smithi114.stdout: python3-rados x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 320 k 2024-04-12T08:33:04.875 INFO:teuthology.orchestra.run.smithi114.stdout: python3-rbd x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 292 k 2024-04-12T08:33:04.875 INFO:teuthology.orchestra.run.smithi114.stdout: python3-rgw x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 99 k 2024-04-12T08:33:04.875 INFO:teuthology.orchestra.run.smithi114.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-12T08:33:04.875 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:04.875 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:33:04.875 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:04.875 INFO:teuthology.orchestra.run.smithi114.stdout:Install 22 Packages 2024-04-12T08:33:04.875 INFO:teuthology.orchestra.run.smithi114.stdout:Upgrade 2 Packages 2024-04-12T08:33:04.875 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:04.876 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 58 M 2024-04-12T08:33:04.876 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:33:06.038 INFO:teuthology.orchestra.run.smithi114.stdout:(1/24): ceph-base-17.2.7-827.gd3b5060a.el9.x86_ 6.3 MB/s | 5.0 MB 00:00 2024-04-12T08:33:06.113 INFO:teuthology.orchestra.run.smithi114.stdout:(2/24): ceph-selinux-17.2.7-827.gd3b5060a.el9.x 335 kB/s | 25 kB 00:00 2024-04-12T08:33:06.273 INFO:teuthology.orchestra.run.smithi114.stdout:(3/24): libcephfs2-17.2.7-827.gd3b5060a.el9.x86 3.9 MB/s | 637 kB 00:00 2024-04-12T08:33:06.372 INFO:teuthology.orchestra.run.smithi114.stdout:(4/24): libradosstriper1-17.2.7-827.gd3b5060a.e 4.5 MB/s | 458 kB 00:00 2024-04-12T08:33:06.956 INFO:teuthology.orchestra.run.smithi114.stdout:(5/24): librgw2-17.2.7-827.gd3b5060a.el9.x86_64 8.4 MB/s | 4.9 MB 00:00 2024-04-12T08:33:06.998 INFO:teuthology.orchestra.run.smithi114.stdout:(6/24): python3-ceph-argparse-17.2.7-827.gd3b50 1.1 MB/s | 45 kB 00:00 2024-04-12T08:33:07.056 INFO:teuthology.orchestra.run.smithi114.stdout:(7/24): python3-ceph-common-17.2.7-827.gd3b5060 2.0 MB/s | 117 kB 00:00 2024-04-12T08:33:07.115 INFO:teuthology.orchestra.run.smithi114.stdout:(8/24): python3-cephfs-17.2.7-827.gd3b5060a.el9 2.6 MB/s | 156 kB 00:00 2024-04-12T08:33:07.190 INFO:teuthology.orchestra.run.smithi114.stdout:(9/24): python3-rados-17.2.7-827.gd3b5060a.el9. 4.2 MB/s | 320 kB 00:00 2024-04-12T08:33:07.257 INFO:teuthology.orchestra.run.smithi114.stdout:(10/24): python3-rbd-17.2.7-827.gd3b5060a.el9.x 4.3 MB/s | 292 kB 00:00 2024-04-12T08:33:07.325 INFO:teuthology.orchestra.run.smithi114.stdout:(11/24): python3-rgw-17.2.7-827.gd3b5060a.el9.x 1.5 MB/s | 99 kB 00:00 2024-04-12T08:33:07.408 INFO:teuthology.orchestra.run.smithi114.stdout:(12/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.9 MB/s | 160 kB 00:00 2024-04-12T08:33:07.467 INFO:teuthology.orchestra.run.smithi114.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 777 kB/s | 45 kB 00:00 2024-04-12T08:33:07.550 INFO:teuthology.orchestra.run.smithi114.stdout:(14/24): librdkafka-1.6.1-102.el9.x86_64.rpm 7.8 MB/s | 662 kB 00:00 2024-04-12T08:33:07.626 INFO:teuthology.orchestra.run.smithi114.stdout:(15/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.8 MB/s | 292 kB 00:00 2024-04-12T08:33:07.684 INFO:teuthology.orchestra.run.smithi114.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 727 kB/s | 42 kB 00:00 2024-04-12T08:33:07.993 INFO:teuthology.orchestra.run.smithi114.stdout:(17/24): ceph-common-17.2.7-827.gd3b5060a.el9.x 8.2 MB/s | 22 MB 00:02 2024-04-12T08:33:08.243 INFO:teuthology.orchestra.run.smithi114.stdout:(18/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 553 kB/s | 309 kB 00:00 2024-04-12T08:33:08.318 INFO:teuthology.orchestra.run.smithi114.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 149 kB/s | 49 kB 00:00 2024-04-12T08:33:08.343 INFO:teuthology.orchestra.run.smithi114.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 672 kB/s | 67 kB 00:00 2024-04-12T08:33:08.735 INFO:teuthology.orchestra.run.smithi114.stdout:(21/24): librados2-17.2.7-827.gd3b5060a.el9.x86 7.9 MB/s | 3.1 MB 00:00 2024-04-12T08:33:08.777 INFO:teuthology.orchestra.run.smithi114.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 3.4 MB/s | 1.6 MB 00:00 2024-04-12T08:33:08.994 INFO:teuthology.orchestra.run.smithi114.stdout:(23/24): librbd1-17.2.7-827.gd3b5060a.el9.x86_6 11 MB/s | 2.9 MB 00:00 2024-04-12T08:33:09.261 INFO:teuthology.orchestra.run.smithi114.stdout:(24/24): ceph-radosgw-17.2.7-827.gd3b5060a.el9. 3.5 MB/s | 14 MB 00:04 2024-04-12T08:33:09.263 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:33:09.264 INFO:teuthology.orchestra.run.smithi114.stdout:Total 13 MB/s | 58 MB 00:04 2024-04-12T08:33:09.323 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:33:09.355 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:33:09.355 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:33:09.757 INFO:teuthology.orchestra.run.smithi090.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-04-12T08:33:09.829 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:33:09.830 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:33:11.111 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:33:11.445 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-12T08:33:11.513 INFO:teuthology.orchestra.run.smithi114.stdout: Upgrading : librados2-2:17.2.7-827.gd3b5060a.el9.x86_64 2/26 2024-04-12T08:33:11.588 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: librados2-2:17.2.7-827.gd3b5060a.el9.x86_64 2/26 2024-04-12T08:33:11.666 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-rados-2:17.2.7-827.gd3b5060a.el9.x86_64 3/26 2024-04-12T08:33:11.698 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : libcephfs2-2:17.2.7-827.gd3b5060a.el9.x86_64 4/26 2024-04-12T08:33:11.788 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: libcephfs2-2:17.2.7-827.gd3b5060a.el9.x86_64 4/26 2024-04-12T08:33:11.834 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-12T08:33:11.872 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-12T08:33:12.303 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-12T08:33:12.328 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : librgw2-2:17.2.7-827.gd3b5060a.el9.x86_64 8/26 2024-04-12T08:33:12.331 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:33:12.331 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:33:12.331 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:33:12.331 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.331 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:33:12.331 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.332 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.333 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:33:12.334 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:33:12.335 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.336 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.337 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.338 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.339 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:33:12.339 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.339 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.339 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.339 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.339 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:33:12.339 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.339 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.339 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.339 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.340 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:33:12.340 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.341 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.341 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.341 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.341 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:33:12.341 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.341 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.341 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.341 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:33:12.391 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: librgw2-2:17.2.7-827.gd3b5060a.el9.x86_64 8/26 2024-04-12T08:33:12.419 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-12T08:33:12.461 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-ceph-argparse-2:17.2.7-827.gd3b5060a.el9.x 10/26 2024-04-12T08:33:12.499 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-cephfs-2:17.2.7-827.gd3b5060a.el9.x86_64 11/26 2024-04-12T08:33:12.561 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-rgw-2:17.2.7-827.gd3b5060a.el9.x86_64 12/26 2024-04-12T08:33:12.589 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : libradosstriper1-2:17.2.7-827.gd3b5060a.el9.x86_64 13/26 2024-04-12T08:33:12.638 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: libradosstriper1-2:17.2.7-827.gd3b5060a.el9.x86_64 13/26 2024-04-12T08:33:12.692 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-12T08:33:12.730 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-12T08:33:12.763 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-12T08:33:12.855 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:33:12.867 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:33:12.867 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-12T08:33:12.867 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:33:12.867 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:33:12.867 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-radosgw x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 14 M 2024-04-12T08:33:12.867 INFO:teuthology.orchestra.run.smithi090.stdout:Upgrading: 2024-04-12T08:33:12.867 INFO:teuthology.orchestra.run.smithi090.stdout: librados2 x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 3.1 M 2024-04-12T08:33:12.867 INFO:teuthology.orchestra.run.smithi090.stdout: librbd1 x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 2.9 M 2024-04-12T08:33:12.867 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-12T08:33:12.867 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-base x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 5.0 M 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-common x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 22 M 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-selinux x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 25 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: libcephfs2 x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 637 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: libradosstriper1 x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 458 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: librgw2 x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 4.9 M 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: python3-ceph-argparse x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 45 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: python3-ceph-common x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 117 k 2024-04-12T08:33:12.868 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cephfs x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 156 k 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rados x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 320 k 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rbd x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 292 k 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rgw x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 99 k 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout:Install 22 Packages 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout:Upgrade 2 Packages 2024-04-12T08:33:12.869 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:12.870 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 58 M 2024-04-12T08:33:12.870 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:33:13.065 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-12T08:33:13.093 INFO:teuthology.orchestra.run.smithi114.stdout: Upgrading : librbd1-2:17.2.7-827.gd3b5060a.el9.x86_64 18/26 2024-04-12T08:33:13.183 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: librbd1-2:17.2.7-827.gd3b5060a.el9.x86_64 18/26 2024-04-12T08:33:13.255 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-rbd-2:17.2.7-827.gd3b5060a.el9.x86_64 19/26 2024-04-12T08:33:13.278 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-ceph-common-2:17.2.7-827.gd3b5060a.el9.x86 20/26 2024-04-12T08:33:14.409 INFO:teuthology.orchestra.run.smithi090.stdout:(1/24): ceph-base-17.2.7-827.gd3b5060a.el9.x86_ 4.5 MB/s | 5.0 MB 00:01 2024-04-12T08:33:14.493 INFO:teuthology.orchestra.run.smithi090.stdout:(2/24): ceph-selinux-17.2.7-827.gd3b5060a.el9.x 301 kB/s | 25 kB 00:00 2024-04-12T08:33:14.684 INFO:teuthology.orchestra.run.smithi090.stdout:(3/24): libcephfs2-17.2.7-827.gd3b5060a.el9.x86 3.3 MB/s | 637 kB 00:00 2024-04-12T08:33:14.912 INFO:teuthology.orchestra.run.smithi090.stdout:(4/24): libradosstriper1-17.2.7-827.gd3b5060a.e 2.0 MB/s | 458 kB 00:00 2024-04-12T08:33:15.169 INFO:teuthology.orchestra.run.smithi090.stdout:(5/24): ceph-common-17.2.7-827.gd3b5060a.el9.x8 12 MB/s | 22 MB 00:01 2024-04-12T08:33:15.229 INFO:teuthology.orchestra.run.smithi090.stdout:(6/24): python3-ceph-argparse-17.2.7-827.gd3b50 758 kB/s | 45 kB 00:00 2024-04-12T08:33:15.287 INFO:teuthology.orchestra.run.smithi090.stdout:(7/24): python3-ceph-common-17.2.7-827.gd3b5060 2.0 MB/s | 117 kB 00:00 2024-04-12T08:33:15.513 INFO:teuthology.orchestra.run.smithi090.stdout:(8/24): ceph-radosgw-17.2.7-827.gd3b5060a.el9.x 6.4 MB/s | 14 MB 00:02 2024-04-12T08:33:15.588 INFO:teuthology.orchestra.run.smithi090.stdout:(9/24): librgw2-17.2.7-827.gd3b5060a.el9.x86_64 7.3 MB/s | 4.9 MB 00:00 2024-04-12T08:33:15.607 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 21/26 2024-04-12T08:33:15.613 INFO:teuthology.orchestra.run.smithi090.stdout:(10/24): python3-cephfs-17.2.7-827.gd3b5060a.el 479 kB/s | 156 kB 00:00 2024-04-12T08:33:15.647 INFO:teuthology.orchestra.run.smithi090.stdout:(11/24): python3-rados-17.2.7-827.gd3b5060a.el9 2.3 MB/s | 320 kB 00:00 2024-04-12T08:33:15.650 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 21/26 2024-04-12T08:33:15.672 INFO:teuthology.orchestra.run.smithi090.stdout:(12/24): python3-rgw-17.2.7-827.gd3b5060a.el9.x 1.7 MB/s | 99 kB 00:00 2024-04-12T08:33:15.697 INFO:teuthology.orchestra.run.smithi090.stdout:(13/24): python3-rbd-17.2.7-827.gd3b5060a.el9.x 2.6 MB/s | 292 kB 00:00 2024-04-12T08:33:15.756 INFO:teuthology.orchestra.run.smithi090.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.4 MB/s | 160 kB 00:00 2024-04-12T08:33:15.781 INFO:teuthology.orchestra.run.smithi090.stdout:(15/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 415 kB/s | 45 kB 00:00 2024-04-12T08:33:15.857 INFO:teuthology.orchestra.run.smithi090.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 562 kB/s | 42 kB 00:00 2024-04-12T08:33:15.890 INFO:teuthology.orchestra.run.smithi090.stdout:(17/24): librdkafka-1.6.1-102.el9.x86_64.rpm 3.4 MB/s | 662 kB 00:00 2024-04-12T08:33:15.916 INFO:teuthology.orchestra.run.smithi090.stdout:(18/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.8 MB/s | 292 kB 00:00 2024-04-12T08:33:16.184 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 21/26 2024-04-12T08:33:16.228 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-base-2:17.2.7-827.gd3b5060a.el9.x86_64 22/26 2024-04-12T08:33:16.326 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-base-2:17.2.7-827.gd3b5060a.el9.x86_64 22/26 2024-04-12T08:33:16.326 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-12T08:33:16.326 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-12T08:33:16.326 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:16.341 INFO:teuthology.orchestra.run.smithi090.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 108 kB/s | 49 kB 00:00 2024-04-12T08:33:16.355 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-selinux-2:17.2.7-827.gd3b5060a.el9.x86_64 23/26 2024-04-12T08:33:16.391 INFO:teuthology.orchestra.run.smithi090.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 142 kB/s | 67 kB 00:00 2024-04-12T08:33:16.508 INFO:teuthology.orchestra.run.smithi090.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 474 kB/s | 309 kB 00:00 2024-04-12T08:33:16.767 INFO:teuthology.orchestra.run.smithi090.stdout:(22/24): librados2-17.2.7-827.gd3b5060a.el9.x86 8.2 MB/s | 3.1 MB 00:00 2024-04-12T08:33:16.826 INFO:teuthology.orchestra.run.smithi090.stdout:(23/24): librbd1-17.2.7-827.gd3b5060a.el9.x86_6 9.3 MB/s | 2.9 MB 00:00 2024-04-12T08:33:16.892 INFO:teuthology.orchestra.run.smithi090.stdout:(24/24): thrift-0.15.0-2.el9.x86_64.rpm 2.9 MB/s | 1.6 MB 00:00 2024-04-12T08:33:16.898 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:33:16.898 INFO:teuthology.orchestra.run.smithi090.stdout:Total 14 MB/s | 58 MB 00:04 2024-04-12T08:33:16.956 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:33:16.990 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:33:16.990 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:33:17.448 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:33:17.449 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:33:18.646 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:33:18.985 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-12T08:33:19.055 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : librados2-2:17.2.7-827.gd3b5060a.el9.x86_64 2/26 2024-04-12T08:33:19.138 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librados2-2:17.2.7-827.gd3b5060a.el9.x86_64 2/26 2024-04-12T08:33:19.225 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-rados-2:17.2.7-827.gd3b5060a.el9.x86_64 3/26 2024-04-12T08:33:19.256 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libcephfs2-2:17.2.7-827.gd3b5060a.el9.x86_64 4/26 2024-04-12T08:33:19.344 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libcephfs2-2:17.2.7-827.gd3b5060a.el9.x86_64 4/26 2024-04-12T08:33:19.400 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-12T08:33:19.445 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-12T08:33:19.896 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-12T08:33:19.927 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : librgw2-2:17.2.7-827.gd3b5060a.el9.x86_64 8/26 2024-04-12T08:33:19.989 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librgw2-2:17.2.7-827.gd3b5060a.el9.x86_64 8/26 2024-04-12T08:33:20.026 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-12T08:33:20.069 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-ceph-argparse-2:17.2.7-827.gd3b5060a.el9.x 10/26 2024-04-12T08:33:20.106 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cephfs-2:17.2.7-827.gd3b5060a.el9.x86_64 11/26 2024-04-12T08:33:20.177 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-rgw-2:17.2.7-827.gd3b5060a.el9.x86_64 12/26 2024-04-12T08:33:20.206 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libradosstriper1-2:17.2.7-827.gd3b5060a.el9.x86_64 13/26 2024-04-12T08:33:20.255 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libradosstriper1-2:17.2.7-827.gd3b5060a.el9.x86_64 13/26 2024-04-12T08:33:20.294 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-12T08:33:20.331 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-12T08:33:20.364 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-12T08:33:20.658 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-12T08:33:20.686 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : librbd1-2:17.2.7-827.gd3b5060a.el9.x86_64 18/26 2024-04-12T08:33:20.775 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librbd1-2:17.2.7-827.gd3b5060a.el9.x86_64 18/26 2024-04-12T08:33:20.849 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-rbd-2:17.2.7-827.gd3b5060a.el9.x86_64 19/26 2024-04-12T08:33:20.880 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-ceph-common-2:17.2.7-827.gd3b5060a.el9.x86 20/26 2024-04-12T08:33:23.264 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 21/26 2024-04-12T08:33:23.329 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 21/26 2024-04-12T08:33:23.886 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 21/26 2024-04-12T08:33:23.925 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-base-2:17.2.7-827.gd3b5060a.el9.x86_64 22/26 2024-04-12T08:33:24.016 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-base-2:17.2.7-827.gd3b5060a.el9.x86_64 22/26 2024-04-12T08:33:24.016 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-12T08:33:24.016 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-12T08:33:24.016 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:24.060 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-selinux-2:17.2.7-827.gd3b5060a.el9.x86_64 23/26 2024-04-12T08:33:25.840 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-selinux-2:17.2.7-827.gd3b5060a.el9.x86_64 23/26 2024-04-12T08:33:25.840 INFO:teuthology.orchestra.run.smithi114.stdout:skipping the directory /sys 2024-04-12T08:33:25.840 INFO:teuthology.orchestra.run.smithi114.stdout:skipping the directory /proc 2024-04-12T08:33:25.840 INFO:teuthology.orchestra.run.smithi114.stdout:skipping the directory /mnt 2024-04-12T08:33:25.841 INFO:teuthology.orchestra.run.smithi114.stdout:skipping the directory /var/tmp 2024-04-12T08:33:25.841 INFO:teuthology.orchestra.run.smithi114.stdout:skipping the directory /home 2024-04-12T08:33:25.841 INFO:teuthology.orchestra.run.smithi114.stdout:skipping the directory /root 2024-04-12T08:33:25.841 INFO:teuthology.orchestra.run.smithi114.stdout:skipping the directory /tmp 2024-04-12T08:33:25.841 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:27.146 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-radosgw-2:17.2.7-827.gd3b5060a.el9.x86_64 24/26 2024-04-12T08:33:27.224 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-827.gd3b5060a.el9.x86_64 24/26 2024-04-12T08:33:27.224 INFO:teuthology.orchestra.run.smithi114.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:33:27.224 INFO:teuthology.orchestra.run.smithi114.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-12T08:33:27.224 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-12T08:33:27.224 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-12T08:33:27.224 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:27.226 INFO:teuthology.orchestra.run.smithi114.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-12T08:33:27.316 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-12T08:33:27.316 INFO:teuthology.orchestra.run.smithi114.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-12T08:33:28.452 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-12T08:33:28.452 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-base-2:17.2.7-827.gd3b5060a.el9.x86_64 1/26 2024-04-12T08:33:28.452 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 2/26 2024-04-12T08:33:28.452 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-radosgw-2:17.2.7-827.gd3b5060a.el9.x86_64 3/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-selinux-2:17.2.7-827.gd3b5060a.el9.x86_64 4/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : libcephfs2-2:17.2.7-827.gd3b5060a.el9.x86_64 5/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : libradosstriper1-2:17.2.7-827.gd3b5060a.el9.x86_64 6/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : librgw2-2:17.2.7-827.gd3b5060a.el9.x86_64 7/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-ceph-argparse-2:17.2.7-827.gd3b5060a.el9.x 8/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-ceph-common-2:17.2.7-827.gd3b5060a.el9.x86 9/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-cephfs-2:17.2.7-827.gd3b5060a.el9.x86_64 10/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-rados-2:17.2.7-827.gd3b5060a.el9.x86_64 11/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-rbd-2:17.2.7-827.gd3b5060a.el9.x86_64 12/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-rgw-2:17.2.7-827.gd3b5060a.el9.x86_64 13/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-12T08:33:28.453 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-12T08:33:28.454 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-12T08:33:28.454 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-12T08:33:28.454 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-12T08:33:28.454 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-12T08:33:28.454 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : librados2-2:17.2.7-827.gd3b5060a.el9.x86_64 23/26 2024-04-12T08:33:28.454 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-12T08:33:28.454 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : librbd1-2:17.2.7-827.gd3b5060a.el9.x86_64 25/26 2024-04-12T08:33:29.513 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout:Upgraded: 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout: librados2-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout: librbd1-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-base-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-radosgw-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-selinux-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-12T08:33:29.514 INFO:teuthology.orchestra.run.smithi114.stdout: libcephfs2-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: libradosstriper1-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: librgw2-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: python3-ceph-argparse-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: python3-ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cephfs-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: python3-rados-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.515 INFO:teuthology.orchestra.run.smithi114.stdout: python3-rbd-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.516 INFO:teuthology.orchestra.run.smithi114.stdout: python3-rgw-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:29.516 INFO:teuthology.orchestra.run.smithi114.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-12T08:33:29.516 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:29.516 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:33:29.892 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph-test 2024-04-12T08:33:30.520 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:00:29 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:33:30.805 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:33:30.805 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:33:30.805 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:33:30.805 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.805 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:33:30.805 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.805 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:33:30.806 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:33:30.807 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:33:30.808 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:33:30.809 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:33:30.810 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.811 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:33:30.811 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.811 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:33:30.811 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.811 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:33:30.811 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.811 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.811 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.811 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.812 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.813 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.814 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.815 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.815 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:33:30.880 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repository Size 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-test x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 48 M 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout:Installing dependencies: 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout: libcephsqlite x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 163 k 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout: oniguruma x86_64 6.9.6-1.el9.5 baseos 218 k 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-12T08:33:30.881 INFO:teuthology.orchestra.run.smithi114.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-12T08:33:30.882 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:30.882 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:33:30.882 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:30.882 INFO:teuthology.orchestra.run.smithi114.stdout:Install 6 Packages 2024-04-12T08:33:30.882 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:30.882 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 49 M 2024-04-12T08:33:30.883 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 193 M 2024-04-12T08:33:30.883 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:33:31.565 INFO:teuthology.orchestra.run.smithi114.stdout:(1/6): jq-1.6-16.el9.x86_64.rpm 1.4 MB/s | 187 kB 00:00 2024-04-12T08:33:31.646 INFO:teuthology.orchestra.run.smithi114.stdout:(2/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 2.6 MB/s | 218 kB 00:00 2024-04-12T08:33:31.822 INFO:teuthology.orchestra.run.smithi114.stdout:(3/6): libcephsqlite-17.2.7-827.gd3b5060a.el9.x 420 kB/s | 163 kB 00:00 2024-04-12T08:33:32.481 INFO:teuthology.orchestra.run.smithi114.stdout:(4/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 97 kB/s | 64 kB 00:00 2024-04-12T08:33:32.697 INFO:teuthology.orchestra.run.smithi114.stdout:(5/6): socat-1.7.4.1-5.el9.x86_64.rpm 290 kB/s | 305 kB 00:01 2024-04-12T08:33:34.343 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-selinux-2:17.2.7-827.gd3b5060a.el9.x86_64 23/26 2024-04-12T08:33:34.343 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /sys 2024-04-12T08:33:34.343 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /proc 2024-04-12T08:33:34.343 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /mnt 2024-04-12T08:33:34.343 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /var/tmp 2024-04-12T08:33:34.343 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /home 2024-04-12T08:33:34.343 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /root 2024-04-12T08:33:34.343 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /tmp 2024-04-12T08:33:34.343 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:35.618 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-radosgw-2:17.2.7-827.gd3b5060a.el9.x86_64 24/26 2024-04-12T08:33:35.685 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-827.gd3b5060a.el9.x86_64 24/26 2024-04-12T08:33:35.685 INFO:teuthology.orchestra.run.smithi090.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:33:35.686 INFO:teuthology.orchestra.run.smithi090.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-12T08:33:35.686 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-12T08:33:35.686 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-12T08:33:35.686 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:35.687 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-12T08:33:35.780 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-12T08:33:35.780 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-12T08:33:36.030 INFO:teuthology.orchestra.run.smithi114.stdout:(6/6): ceph-test-17.2.7-827.gd3b5060a.el9.x86_6 10 MB/s | 48 MB 00:04 2024-04-12T08:33:36.032 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:33:36.033 INFO:teuthology.orchestra.run.smithi114.stdout:Total 9.5 MB/s | 49 MB 00:05 2024-04-12T08:33:36.084 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:33:36.104 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:33:36.104 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:33:36.486 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:33:36.486 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:33:37.063 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:33:37.149 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-12T08:33:37.190 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-12T08:33:37.231 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-12T08:33:37.273 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-12T08:33:37.290 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-12T08:33:37.290 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-base-2:17.2.7-827.gd3b5060a.el9.x86_64 1/26 2024-04-12T08:33:37.290 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 2/26 2024-04-12T08:33:37.290 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-radosgw-2:17.2.7-827.gd3b5060a.el9.x86_64 3/26 2024-04-12T08:33:37.290 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-selinux-2:17.2.7-827.gd3b5060a.el9.x86_64 4/26 2024-04-12T08:33:37.290 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libcephfs2-2:17.2.7-827.gd3b5060a.el9.x86_64 5/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libradosstriper1-2:17.2.7-827.gd3b5060a.el9.x86_64 6/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librgw2-2:17.2.7-827.gd3b5060a.el9.x86_64 7/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-ceph-argparse-2:17.2.7-827.gd3b5060a.el9.x 8/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-ceph-common-2:17.2.7-827.gd3b5060a.el9.x86 9/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cephfs-2:17.2.7-827.gd3b5060a.el9.x86_64 10/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-rados-2:17.2.7-827.gd3b5060a.el9.x86_64 11/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-rbd-2:17.2.7-827.gd3b5060a.el9.x86_64 12/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-rgw-2:17.2.7-827.gd3b5060a.el9.x86_64 13/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-12T08:33:37.291 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-12T08:33:37.292 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-12T08:33:37.292 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-12T08:33:37.292 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-12T08:33:37.292 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librados2-2:17.2.7-827.gd3b5060a.el9.x86_64 23/26 2024-04-12T08:33:37.292 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-12T08:33:37.292 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librbd1-2:17.2.7-827.gd3b5060a.el9.x86_64 25/26 2024-04-12T08:33:37.300 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : libcephsqlite-2:17.2.7-827.gd3b5060a.el9.x86_64 5/6 2024-04-12T08:33:38.292 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-12T08:33:38.292 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:38.292 INFO:teuthology.orchestra.run.smithi090.stdout:Upgraded: 2024-04-12T08:33:38.292 INFO:teuthology.orchestra.run.smithi090.stdout: librados2-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: librbd1-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-base-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-radosgw-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-selinux-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: libcephfs2-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: libradosstriper1-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: librgw2-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.293 INFO:teuthology.orchestra.run.smithi090.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout: python3-ceph-argparse-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout: python3-ceph-common-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cephfs-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rados-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rbd-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rgw-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:38.294 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:33:38.724 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-test 2024-04-12T08:33:39.307 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:00:30 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:33:39.591 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:33:39.592 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.593 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:33:39.594 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.595 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.596 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.597 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:33:39.598 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.599 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.600 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.601 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:33:39.690 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:33:39.691 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:33:39.691 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-12T08:33:39.691 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:33:39.691 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:33:39.691 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-test x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 48 M 2024-04-12T08:33:39.691 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-12T08:33:39.691 INFO:teuthology.orchestra.run.smithi090.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-12T08:33:39.691 INFO:teuthology.orchestra.run.smithi090.stdout: libcephsqlite x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 163 k 2024-04-12T08:33:39.692 INFO:teuthology.orchestra.run.smithi090.stdout: oniguruma x86_64 6.9.6-1.el9.5 baseos 218 k 2024-04-12T08:33:39.692 INFO:teuthology.orchestra.run.smithi090.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-12T08:33:39.692 INFO:teuthology.orchestra.run.smithi090.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-12T08:33:39.692 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:39.692 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:33:39.692 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:33:39.692 INFO:teuthology.orchestra.run.smithi090.stdout:Install 6 Packages 2024-04-12T08:33:39.692 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:39.693 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 49 M 2024-04-12T08:33:39.693 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 193 M 2024-04-12T08:33:39.693 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:33:40.169 INFO:teuthology.orchestra.run.smithi090.stdout:(1/6): jq-1.6-16.el9.x86_64.rpm 1.1 MB/s | 187 kB 00:00 2024-04-12T08:33:40.236 INFO:teuthology.orchestra.run.smithi090.stdout:(2/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 3.2 MB/s | 218 kB 00:00 2024-04-12T08:33:40.370 INFO:teuthology.orchestra.run.smithi090.stdout:(3/6): socat-1.7.4.1-5.el9.x86_64.rpm 2.2 MB/s | 305 kB 00:00 2024-04-12T08:33:40.412 INFO:teuthology.orchestra.run.smithi090.stdout:(4/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-04-12T08:33:40.437 INFO:teuthology.orchestra.run.smithi090.stdout:(5/6): libcephsqlite-17.2.7-827.gd3b5060a.el9.x 376 kB/s | 163 kB 00:00 2024-04-12T08:33:41.728 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: libcephsqlite-2:17.2.7-827.gd3b5060a.el9.x86_64 5/6 2024-04-12T08:33:41.816 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-test-2:17.2.7-827.gd3b5060a.el9.x86_64 6/6 2024-04-12T08:33:42.721 INFO:teuthology.orchestra.run.smithi090.stdout:(6/6): ceph-test-17.2.7-827.gd3b5060a.el9.x86_6 18 MB/s | 48 MB 00:02 2024-04-12T08:33:42.722 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:33:42.723 INFO:teuthology.orchestra.run.smithi090.stdout:Total 16 MB/s | 49 MB 00:03 2024-04-12T08:33:42.774 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:33:42.794 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:33:42.794 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:33:43.183 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:33:43.183 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:33:43.781 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:33:43.868 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-12T08:33:43.906 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-12T08:33:43.941 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-12T08:33:43.980 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-12T08:33:44.016 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libcephsqlite-2:17.2.7-827.gd3b5060a.el9.x86_64 5/6 2024-04-12T08:33:44.056 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-test-2:17.2.7-827.gd3b5060a.el9.x86_64 6/6 2024-04-12T08:33:44.056 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-test-2:17.2.7-827.gd3b5060a.el9.x86_64 1/6 2024-04-12T08:33:44.057 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : libcephsqlite-2:17.2.7-827.gd3b5060a.el9.x86_64 2/6 2024-04-12T08:33:44.057 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : jq-1.6-16.el9.x86_64 3/6 2024-04-12T08:33:44.057 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 4/6 2024-04-12T08:33:44.057 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 5/6 2024-04-12T08:33:44.418 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 6/6 2024-04-12T08:33:44.418 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:44.418 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:33:44.418 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-test-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:44.418 INFO:teuthology.orchestra.run.smithi114.stdout: jq-1.6-16.el9.x86_64 2024-04-12T08:33:44.418 INFO:teuthology.orchestra.run.smithi114.stdout: libcephsqlite-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:44.418 INFO:teuthology.orchestra.run.smithi114.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-12T08:33:44.418 INFO:teuthology.orchestra.run.smithi114.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-12T08:33:44.419 INFO:teuthology.orchestra.run.smithi114.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-12T08:33:44.419 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:44.419 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:33:44.731 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph 2024-04-12T08:33:45.338 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:00:44 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:33:45.636 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:33:45.636 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:33:45.636 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:33:45.636 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.636 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:33:45.636 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.636 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:33:45.637 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:33:45.638 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.639 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.640 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.641 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.642 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:33:45.643 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:33:45.644 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.645 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.645 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.645 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:33:45.715 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:33:45.717 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:45.717 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repository Size 2024-04-12T08:33:45.717 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:45.717 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:33:45.717 INFO:teuthology.orchestra.run.smithi114.stdout: ceph x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 6.5 k 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout:Installing dependencies: 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mds x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 2.1 M 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 1.4 M 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-modules-core noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 240 k 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mon x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 3.9 M 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-osd x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 15 M 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-12T08:33:45.718 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-12T08:33:45.719 INFO:teuthology.orchestra.run.smithi114.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-12T08:33:45.720 INFO:teuthology.orchestra.run.smithi114.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-12T08:33:45.720 INFO:teuthology.orchestra.run.smithi114.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-12T08:33:45.720 INFO:teuthology.orchestra.run.smithi114.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-12T08:33:45.720 INFO:teuthology.orchestra.run.smithi114.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-12T08:33:45.720 INFO:teuthology.orchestra.run.smithi114.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-12T08:33:45.720 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:45.720 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:33:45.720 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:45.720 INFO:teuthology.orchestra.run.smithi114.stdout:Install 33 Packages 2024-04-12T08:33:45.720 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:45.721 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 27 M 2024-04-12T08:33:45.721 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 96 M 2024-04-12T08:33:45.721 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:33:46.333 INFO:teuthology.orchestra.run.smithi114.stdout:(1/33): ceph-17.2.7-827.gd3b5060a.el9.x86_64.rp 19 kB/s | 6.5 kB 00:00 2024-04-12T08:33:46.825 INFO:teuthology.orchestra.run.smithi114.stdout:(2/33): ceph-mgr-17.2.7-827.gd3b5060a.el9.x86_6 1.7 MB/s | 1.4 MB 00:00 2024-04-12T08:33:46.925 INFO:teuthology.orchestra.run.smithi114.stdout:(3/33): ceph-mds-17.2.7-827.gd3b5060a.el9.x86_6 2.2 MB/s | 2.1 MB 00:00 2024-04-12T08:33:46.992 INFO:teuthology.orchestra.run.smithi114.stdout:(4/33): ceph-mgr-modules-core-17.2.7-827.gd3b50 3.5 MB/s | 240 kB 00:00 2024-04-12T08:33:47.134 INFO:teuthology.orchestra.run.smithi114.stdout:(5/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.7 MB/s | 253 kB 00:00 2024-04-12T08:33:47.234 INFO:teuthology.orchestra.run.smithi114.stdout:(6/33): python3-cryptography-36.0.1-4.el9.x86_6 12 MB/s | 1.2 MB 00:00 2024-04-12T08:33:47.284 INFO:teuthology.orchestra.run.smithi114.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 2.1 MB/s | 106 kB 00:00 2024-04-12T08:33:47.326 INFO:teuthology.orchestra.run.smithi114.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 3.2 MB/s | 135 kB 00:00 2024-04-12T08:33:47.377 INFO:teuthology.orchestra.run.smithi114.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 2.4 MB/s | 125 kB 00:00 2024-04-12T08:33:47.427 INFO:teuthology.orchestra.run.smithi114.stdout:(10/33): python3-urllib3-1.26.5-5.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-04-12T08:33:47.469 INFO:teuthology.orchestra.run.smithi114.stdout:(11/33): python3-mako-1.1.4-6.el9.noarch.rpm 4.0 MB/s | 172 kB 00:00 2024-04-12T08:33:47.503 INFO:teuthology.orchestra.run.smithi114.stdout:(12/33): python3-markupsafe-1.1.1-12.el9.x86_64 1.0 MB/s | 35 kB 00:00 2024-04-12T08:33:47.536 INFO:teuthology.orchestra.run.smithi114.stdout:(13/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.5 MB/s | 51 kB 00:00 2024-04-12T08:33:47.570 INFO:teuthology.orchestra.run.smithi114.stdout:(14/33): python3-toml-0.10.2-6.el9.noarch.rpm 1.2 MB/s | 42 kB 00:00 2024-04-12T08:33:47.679 INFO:teuthology.orchestra.run.smithi114.stdout:(15/33): ceph-mon-17.2.7-827.gd3b5060a.el9.x86_ 2.9 MB/s | 3.9 MB 00:01 2024-04-12T08:33:48.057 INFO:teuthology.orchestra.run.smithi114.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 89 kB/s | 43 kB 00:00 2024-04-12T08:33:48.338 INFO:teuthology.orchestra.run.smithi114.stdout:(17/33): ceph-osd-17.2.7-827.gd3b5060a.el9.x86_ 10 MB/s | 15 MB 00:01 2024-04-12T08:33:48.363 INFO:teuthology.orchestra.run.smithi114.stdout:(18/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 260 kB/s | 172 kB 00:00 2024-04-12T08:33:48.463 INFO:teuthology.orchestra.run.smithi114.stdout:(19/33): python3-jaraco-classes-3.2.1-5.el9.noa 177 kB/s | 18 kB 00:00 2024-04-12T08:33:48.473 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libcephsqlite-2:17.2.7-827.gd3b5060a.el9.x86_64 5/6 2024-04-12T08:33:48.513 INFO:teuthology.orchestra.run.smithi114.stdout:(20/33): python3-cherrypy-18.6.1-2.el9.noarch.r 786 kB/s | 358 kB 00:00 2024-04-12T08:33:48.551 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-test-2:17.2.7-827.gd3b5060a.el9.x86_64 6/6 2024-04-12T08:33:48.555 INFO:teuthology.orchestra.run.smithi114.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 251 kB/s | 23 kB 00:00 2024-04-12T08:33:48.605 INFO:teuthology.orchestra.run.smithi114.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 214 kB/s | 19 kB 00:00 2024-04-12T08:33:48.622 INFO:teuthology.orchestra.run.smithi114.stdout:(23/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 38 kB/s | 11 kB 00:00 2024-04-12T08:33:48.648 INFO:teuthology.orchestra.run.smithi114.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 213 kB/s | 20 kB 00:00 2024-04-12T08:33:48.698 INFO:teuthology.orchestra.run.smithi114.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 502 kB/s | 46 kB 00:00 2024-04-12T08:33:48.799 INFO:teuthology.orchestra.run.smithi114.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 164 kB/s | 16 kB 00:00 2024-04-12T08:33:48.840 INFO:teuthology.orchestra.run.smithi114.stdout:(27/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.4 MB/s | 272 kB 00:00 2024-04-12T08:33:48.891 INFO:teuthology.orchestra.run.smithi114.stdout:(28/33): python3-more-itertools-8.12.0-2.el9.no 294 kB/s | 79 kB 00:00 2024-04-12T08:33:48.916 INFO:teuthology.orchestra.run.smithi114.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 768 kB/s | 90 kB 00:00 2024-04-12T08:33:48.941 INFO:teuthology.orchestra.run.smithi114.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 355 kB/s | 36 kB 00:00 2024-04-12T08:33:49.050 INFO:teuthology.orchestra.run.smithi114.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 183 kB/s | 20 kB 00:00 2024-04-12T08:33:49.092 INFO:teuthology.orchestra.run.smithi114.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.4 MB/s | 427 kB 00:00 2024-04-12T08:33:49.142 INFO:teuthology.orchestra.run.smithi114.stdout:(33/33): python3-webob-1.8.7-6.el9.noarch.rpm 915 kB/s | 230 kB 00:00 2024-04-12T08:33:49.149 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:33:49.150 INFO:teuthology.orchestra.run.smithi114.stdout:Total 7.9 MB/s | 27 MB 00:03 2024-04-12T08:33:49.263 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:33:49.299 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:33:49.299 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:33:49.642 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:33:49.643 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:33:50.902 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:33:50.970 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-12T08:33:51.002 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-12T08:33:51.035 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-12T08:33:51.127 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-12T08:33:51.213 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-12T08:33:51.264 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-12T08:33:51.306 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-12T08:33:51.336 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-12T08:33:51.370 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-12T08:33:51.404 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-12T08:33:51.405 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-test-2:17.2.7-827.gd3b5060a.el9.x86_64 6/6 2024-04-12T08:33:51.405 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-test-2:17.2.7-827.gd3b5060a.el9.x86_64 1/6 2024-04-12T08:33:51.405 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libcephsqlite-2:17.2.7-827.gd3b5060a.el9.x86_64 2/6 2024-04-12T08:33:51.405 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : jq-1.6-16.el9.x86_64 3/6 2024-04-12T08:33:51.405 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 4/6 2024-04-12T08:33:51.405 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 5/6 2024-04-12T08:33:51.444 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-12T08:33:51.479 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-12T08:33:51.514 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-12T08:33:51.555 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-12T08:33:51.633 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-12T08:33:51.770 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-12T08:33:51.800 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 6/6 2024-04-12T08:33:51.800 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:51.800 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:33:51.800 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-test-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:51.800 INFO:teuthology.orchestra.run.smithi090.stdout: jq-1.6-16.el9.x86_64 2024-04-12T08:33:51.800 INFO:teuthology.orchestra.run.smithi090.stdout: libcephsqlite-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:51.800 INFO:teuthology.orchestra.run.smithi090.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-12T08:33:51.800 INFO:teuthology.orchestra.run.smithi090.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-12T08:33:51.800 INFO:teuthology.orchestra.run.smithi090.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-12T08:33:51.800 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:51.801 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:33:51.851 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-12T08:33:51.908 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-12T08:33:51.952 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-12T08:33:52.009 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-12T08:33:52.064 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-12T08:33:52.164 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph 2024-04-12T08:33:52.254 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-12T08:33:52.306 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-12T08:33:52.383 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-12T08:33:52.472 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-12T08:33:52.507 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-12T08:33:52.637 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-12T08:33:52.759 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:00:43 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:33:52.786 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-827.gd3b5060a.el9.n 28/33 2024-04-12T08:33:52.816 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-mgr-2:17.2.7-827.gd3b5060a.el9.x86_64 29/33 2024-04-12T08:33:52.841 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-mgr-2:17.2.7-827.gd3b5060a.el9.x86_64 29/33 2024-04-12T08:33:52.841 INFO:teuthology.orchestra.run.smithi114.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:33:52.841 INFO:teuthology.orchestra.run.smithi114.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-12T08:33:52.841 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-12T08:33:52.842 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-12T08:33:52.842 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:53.048 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:33:53.048 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:33:53.048 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:33:53.048 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.048 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:33:53.049 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.049 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:33:53.049 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.049 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:33:53.049 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.049 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:33:53.049 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.049 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:33:53.050 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.050 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:33:53.050 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.050 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:33:53.050 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.050 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:33:53.050 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.051 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:33:53.051 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.051 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:33:53.051 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.051 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:33:53.051 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.051 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:33:53.052 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.052 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:33:53.052 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.052 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:33:53.052 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.052 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:33:53.053 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.053 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:33:53.053 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.053 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:33:53.053 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.054 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:33:53.054 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.054 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:33:53.054 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.054 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:33:53.054 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.054 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:33:53.055 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.055 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:33:53.055 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.055 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:33:53.055 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.055 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:33:53.055 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.055 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:33:53.056 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.056 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:33:53.056 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.056 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:33:53.056 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.056 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:33:53.056 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.056 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:33:53.056 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.056 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.057 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.058 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.058 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:33:53.058 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.058 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.058 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.058 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.058 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:33:53.058 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.058 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.058 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.059 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.060 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.060 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.060 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:33:53.060 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.060 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.060 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.061 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.061 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:33:53.061 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.061 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.061 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.061 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.061 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:33:53.062 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.062 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.062 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.062 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.062 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:33:53.062 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.062 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.062 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.062 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:33:53.153 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:33:53.155 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:33:53.155 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout: ceph x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 6.5 k 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mds x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 2.1 M 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 1.4 M 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-modules-core noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 240 k 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mon x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 3.9 M 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-osd x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 15 M 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-12T08:33:53.156 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-12T08:33:53.157 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-12T08:33:53.158 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:53.159 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:33:53.159 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:33:53.159 INFO:teuthology.orchestra.run.smithi090.stdout:Install 33 Packages 2024-04-12T08:33:53.159 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:53.159 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 27 M 2024-04-12T08:33:53.159 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 96 M 2024-04-12T08:33:53.160 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:33:53.832 INFO:teuthology.orchestra.run.smithi090.stdout:(1/33): ceph-17.2.7-827.gd3b5060a.el9.x86_64.rp 24 kB/s | 6.5 kB 00:00 2024-04-12T08:33:53.923 INFO:teuthology.orchestra.run.smithi090.stdout:(2/33): ceph-mgr-17.2.7-827.gd3b5060a.el9.x86_6 3.8 MB/s | 1.4 MB 00:00 2024-04-12T08:33:54.150 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-osd-2:17.2.7-827.gd3b5060a.el9.x86_64 30/33 2024-04-12T08:33:54.176 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-osd-2:17.2.7-827.gd3b5060a.el9.x86_64 30/33 2024-04-12T08:33:54.177 INFO:teuthology.orchestra.run.smithi114.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:33:54.177 INFO:teuthology.orchestra.run.smithi114.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-12T08:33:54.177 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-12T08:33:54.177 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-12T08:33:54.177 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:54.559 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-mon-2:17.2.7-827.gd3b5060a.el9.x86_64 31/33 2024-04-12T08:33:54.582 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-mon-2:17.2.7-827.gd3b5060a.el9.x86_64 31/33 2024-04-12T08:33:54.583 INFO:teuthology.orchestra.run.smithi114.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:33:54.583 INFO:teuthology.orchestra.run.smithi114.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-12T08:33:54.583 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-12T08:33:54.583 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-12T08:33:54.583 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:54.724 INFO:teuthology.orchestra.run.smithi090.stdout:(3/33): ceph-osd-17.2.7-827.gd3b5060a.el9.x86_6 19 MB/s | 15 MB 00:00 2024-04-12T08:33:54.774 INFO:teuthology.orchestra.run.smithi090.stdout:(4/33): ceph-mgr-modules-core-17.2.7-827.gd3b50 4.7 MB/s | 240 kB 00:00 2024-04-12T08:33:54.784 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-mds-2:17.2.7-827.gd3b5060a.el9.x86_64 32/33 2024-04-12T08:33:54.808 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-mds-2:17.2.7-827.gd3b5060a.el9.x86_64 32/33 2024-04-12T08:33:54.808 INFO:teuthology.orchestra.run.smithi114.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:33:54.808 INFO:teuthology.orchestra.run.smithi114.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-12T08:33:54.808 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-12T08:33:54.808 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-12T08:33:54.808 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:54.873 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-2:17.2.7-827.gd3b5060a.el9.x86_64 33/33 2024-04-12T08:33:54.891 INFO:teuthology.orchestra.run.smithi090.stdout:(5/33): ceph-mds-17.2.7-827.gd3b5060a.el9.x86_6 1.6 MB/s | 2.1 MB 00:01 2024-04-12T08:33:54.958 INFO:teuthology.orchestra.run.smithi090.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.3 MB/s | 253 kB 00:00 2024-04-12T08:33:55.000 INFO:teuthology.orchestra.run.smithi090.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-04-12T08:33:55.051 INFO:teuthology.orchestra.run.smithi090.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 2.6 MB/s | 135 kB 00:00 2024-04-12T08:33:55.093 INFO:teuthology.orchestra.run.smithi090.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 2.9 MB/s | 125 kB 00:00 2024-04-12T08:33:55.135 INFO:teuthology.orchestra.run.smithi090.stdout:(10/33): python3-cryptography-36.0.1-4.el9.x86_ 5.0 MB/s | 1.2 MB 00:00 2024-04-12T08:33:55.168 INFO:teuthology.orchestra.run.smithi090.stdout:(11/33): python3-urllib3-1.26.5-5.el9.noarch.rp 2.8 MB/s | 215 kB 00:00 2024-04-12T08:33:55.352 INFO:teuthology.orchestra.run.smithi090.stdout:(12/33): python3-mako-1.1.4-6.el9.noarch.rpm 793 kB/s | 172 kB 00:00 2024-04-12T08:33:55.369 INFO:teuthology.orchestra.run.smithi090.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 173 kB/s | 35 kB 00:00 2024-04-12T08:33:55.411 INFO:teuthology.orchestra.run.smithi090.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 868 kB/s | 51 kB 00:00 2024-04-12T08:33:55.446 INFO:teuthology.orchestra.run.smithi090.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 542 kB/s | 42 kB 00:00 2024-04-12T08:33:55.521 INFO:teuthology.orchestra.run.smithi090.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 395 kB/s | 43 kB 00:00 2024-04-12T08:33:55.580 INFO:teuthology.orchestra.run.smithi090.stdout:(17/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.3 MB/s | 172 kB 00:00 2024-04-12T08:33:55.614 INFO:teuthology.orchestra.run.smithi090.stdout:(18/33): python3-cherrypy-18.6.1-2.el9.noarch.r 3.8 MB/s | 358 kB 00:00 2024-04-12T08:33:55.639 INFO:teuthology.orchestra.run.smithi090.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 182 kB/s | 11 kB 00:00 2024-04-12T08:33:55.664 INFO:teuthology.orchestra.run.smithi090.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 350 kB/s | 18 kB 00:00 2024-04-12T08:33:55.690 INFO:teuthology.orchestra.run.smithi090.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 461 kB/s | 23 kB 00:00 2024-04-12T08:33:55.715 INFO:teuthology.orchestra.run.smithi090.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 386 kB/s | 19 kB 00:00 2024-04-12T08:33:55.740 INFO:teuthology.orchestra.run.smithi090.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 390 kB/s | 20 kB 00:00 2024-04-12T08:33:55.807 INFO:teuthology.orchestra.run.smithi090.stdout:(24/33): ceph-mon-17.2.7-827.gd3b5060a.el9.x86_ 2.0 MB/s | 3.9 MB 00:01 2024-04-12T08:33:55.833 INFO:teuthology.orchestra.run.smithi090.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 394 kB/s | 46 kB 00:00 2024-04-12T08:33:55.858 INFO:teuthology.orchestra.run.smithi090.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 672 kB/s | 79 kB 00:00 2024-04-12T08:33:55.883 INFO:teuthology.orchestra.run.smithi090.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 326 kB/s | 16 kB 00:00 2024-04-12T08:33:55.909 INFO:teuthology.orchestra.run.smithi090.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-04-12T08:33:55.934 INFO:teuthology.orchestra.run.smithi090.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 709 kB/s | 36 kB 00:00 2024-04-12T08:33:55.976 INFO:teuthology.orchestra.run.smithi090.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.6 MB/s | 272 kB 00:00 2024-04-12T08:33:56.001 INFO:teuthology.orchestra.run.smithi090.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.4 MB/s | 230 kB 00:00 2024-04-12T08:33:56.027 INFO:teuthology.orchestra.run.smithi090.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 398 kB/s | 20 kB 00:00 2024-04-12T08:33:56.060 INFO:teuthology.orchestra.run.smithi090.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.3 MB/s | 427 kB 00:00 2024-04-12T08:33:56.064 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:33:56.064 INFO:teuthology.orchestra.run.smithi090.stdout:Total 9.3 MB/s | 27 MB 00:02 2024-04-12T08:33:56.181 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:33:56.217 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:33:56.217 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:33:56.510 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-2:17.2.7-827.gd3b5060a.el9.x86_64 33/33 2024-04-12T08:33:56.510 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-2:17.2.7-827.gd3b5060a.el9.x86_64 1/33 2024-04-12T08:33:56.510 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-mds-2:17.2.7-827.gd3b5060a.el9.x86_64 2/33 2024-04-12T08:33:56.510 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-mgr-2:17.2.7-827.gd3b5060a.el9.x86_64 3/33 2024-04-12T08:33:56.510 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-mon-2:17.2.7-827.gd3b5060a.el9.x86_64 4/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-osd-2:17.2.7-827.gd3b5060a.el9.x86_64 5/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-827.gd3b5060a.el9.n 6/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-12T08:33:56.511 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-12T08:33:56.512 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-12T08:33:56.512 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-12T08:33:56.512 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-12T08:33:56.512 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-12T08:33:56.512 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-12T08:33:56.512 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-12T08:33:56.513 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-12T08:33:56.513 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-12T08:33:56.513 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-12T08:33:56.513 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-12T08:33:56.513 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-12T08:33:56.513 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-12T08:33:56.513 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-12T08:33:56.574 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:33:56.574 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:33:57.516 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-12T08:33:57.516 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:57.516 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:33:57.516 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:57.516 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mds-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:57.516 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:57.516 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-modules-core-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:33:57.516 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mon-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:57.516 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-osd-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-12T08:33:57.517 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-ply-3.11-14.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-12T08:33:57.518 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:57.519 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:33:57.827 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph-base 2024-04-12T08:33:57.869 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:33:57.930 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-12T08:33:57.979 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-12T08:33:58.013 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-12T08:33:58.104 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-12T08:33:58.175 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-12T08:33:58.216 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-12T08:33:58.251 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-12T08:33:58.290 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-12T08:33:58.325 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-12T08:33:58.358 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-12T08:33:58.398 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-12T08:33:58.416 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:00:57 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:33:58.442 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-12T08:33:58.477 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-12T08:33:58.517 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-12T08:33:58.597 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:33:58.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:33:58.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:33:58.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:33:58.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:33:58.712 INFO:teuthology.orchestra.run.smithi114.stdout:Package ceph-base-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:33:58.734 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-12T08:33:58.769 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:33:58.770 INFO:teuthology.orchestra.run.smithi114.stdout:Nothing to do. 2024-04-12T08:33:58.770 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:33:58.815 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-12T08:33:58.833 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install cephadm 2024-04-12T08:33:58.870 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-12T08:33:58.908 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-12T08:33:58.966 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-12T08:33:59.022 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-12T08:33:59.216 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-12T08:33:59.265 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-12T08:33:59.334 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-12T08:33:59.421 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:00:58 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:33:59.430 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-12T08:33:59.483 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-12T08:33:59.602 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-12T08:33:59.716 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:33:59.716 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:33:59.716 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:33:59.716 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.716 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:33:59.716 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.716 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:33:59.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.718 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.719 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:33:59.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:33:59.721 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.722 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.723 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.724 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:33:59.725 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.725 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.725 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.725 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:33:59.753 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-827.gd3b5060a.el9.n 28/33 2024-04-12T08:33:59.781 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:33:59.782 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:59.782 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repository Size 2024-04-12T08:33:59.782 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:59.783 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:33:59.783 INFO:teuthology.orchestra.run.smithi114.stdout: cephadm noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 82 k 2024-04-12T08:33:59.783 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:59.783 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:33:59.783 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:33:59.783 INFO:teuthology.orchestra.run.smithi114.stdout:Install 1 Package 2024-04-12T08:33:59.783 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:33:59.783 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 82 k 2024-04-12T08:33:59.783 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 372 k 2024-04-12T08:33:59.783 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:33:59.785 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-2:17.2.7-827.gd3b5060a.el9.x86_64 29/33 2024-04-12T08:33:59.811 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mgr-2:17.2.7-827.gd3b5060a.el9.x86_64 29/33 2024-04-12T08:33:59.811 INFO:teuthology.orchestra.run.smithi090.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:33:59.811 INFO:teuthology.orchestra.run.smithi090.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-12T08:33:59.811 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-12T08:33:59.811 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-12T08:33:59.811 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:33:59.977 INFO:teuthology.orchestra.run.smithi114.stdout:cephadm-17.2.7-827.gd3b5060a.el9.noarch.rpm 425 kB/s | 82 kB 00:00 2024-04-12T08:33:59.977 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:33:59.977 INFO:teuthology.orchestra.run.smithi114.stdout:Total 421 kB/s | 82 kB 00:00 2024-04-12T08:33:59.977 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:33:59.982 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:33:59.982 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:33:59.984 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:33:59.984 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:00.066 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:00.432 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 1/1 2024-04-12T08:34:00.522 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 1/1 2024-04-12T08:34:00.896 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 1/1 2024-04-12T08:34:01.178 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-osd-2:17.2.7-827.gd3b5060a.el9.x86_64 30/33 2024-04-12T08:34:01.192 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 1/1 2024-04-12T08:34:01.192 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:01.192 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:01.192 INFO:teuthology.orchestra.run.smithi114.stdout: cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:01.193 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:01.193 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:01.205 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-osd-2:17.2.7-827.gd3b5060a.el9.x86_64 30/33 2024-04-12T08:34:01.205 INFO:teuthology.orchestra.run.smithi090.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:34:01.205 INFO:teuthology.orchestra.run.smithi090.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-12T08:34:01.205 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-12T08:34:01.205 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-12T08:34:01.205 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:01.364 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph-immutable-object-cache 2024-04-12T08:34:01.579 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mon-2:17.2.7-827.gd3b5060a.el9.x86_64 31/33 2024-04-12T08:34:01.603 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mon-2:17.2.7-827.gd3b5060a.el9.x86_64 31/33 2024-04-12T08:34:01.603 INFO:teuthology.orchestra.run.smithi090.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:34:01.603 INFO:teuthology.orchestra.run.smithi090.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-12T08:34:01.603 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-12T08:34:01.603 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-12T08:34:01.603 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:01.806 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mds-2:17.2.7-827.gd3b5060a.el9.x86_64 32/33 2024-04-12T08:34:01.828 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mds-2:17.2.7-827.gd3b5060a.el9.x86_64 32/33 2024-04-12T08:34:01.828 INFO:teuthology.orchestra.run.smithi090.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:34:01.828 INFO:teuthology.orchestra.run.smithi090.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-12T08:34:01.828 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-12T08:34:01.828 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-12T08:34:01.828 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:01.893 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-2:17.2.7-827.gd3b5060a.el9.x86_64 33/33 2024-04-12T08:34:01.973 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:00 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:02.262 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:02.262 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:02.262 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.263 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:02.264 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.265 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:02.266 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:02.267 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.268 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:02.269 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:02.270 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.271 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.271 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.271 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:02.330 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:02.331 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:02.331 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repo Size 2024-04-12T08:34:02.331 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:02.331 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:02.331 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 137 k 2024-04-12T08:34:02.331 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:02.331 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:02.331 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:02.332 INFO:teuthology.orchestra.run.smithi114.stdout:Install 1 Package 2024-04-12T08:34:02.332 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:02.332 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 137 k 2024-04-12T08:34:02.332 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 379 k 2024-04-12T08:34:02.332 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:02.666 INFO:teuthology.orchestra.run.smithi114.stdout:ceph-immutable-object-cache-17.2.7-827.gd3b5060 412 kB/s | 137 kB 00:00 2024-04-12T08:34:02.666 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:02.667 INFO:teuthology.orchestra.run.smithi114.stdout:Total 410 kB/s | 137 kB 00:00 2024-04-12T08:34:02.667 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:02.675 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:02.675 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:02.727 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:02.728 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:02.876 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:02.950 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-827.gd3b5060a.e 1/1 2024-04-12T08:34:02.976 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-827.gd3b5060a.e 1/1 2024-04-12T08:34:02.976 INFO:teuthology.orchestra.run.smithi114.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:34:02.976 INFO:teuthology.orchestra.run.smithi114.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-12T08:34:02.976 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:03.848 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-2:17.2.7-827.gd3b5060a.el9.x86_64 33/33 2024-04-12T08:34:03.848 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-2:17.2.7-827.gd3b5060a.el9.x86_64 1/33 2024-04-12T08:34:03.848 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mds-2:17.2.7-827.gd3b5060a.el9.x86_64 2/33 2024-04-12T08:34:03.848 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-2:17.2.7-827.gd3b5060a.el9.x86_64 3/33 2024-04-12T08:34:03.848 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mon-2:17.2.7-827.gd3b5060a.el9.x86_64 4/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-osd-2:17.2.7-827.gd3b5060a.el9.x86_64 5/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-827.gd3b5060a.el9.n 6/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-12T08:34:03.849 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-12T08:34:03.850 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-12T08:34:03.850 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-12T08:34:03.850 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-12T08:34:03.850 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-12T08:34:03.850 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-12T08:34:03.850 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-12T08:34:03.850 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-12T08:34:03.850 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-12T08:34:03.850 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-12T08:34:03.851 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-12T08:34:03.851 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-12T08:34:03.851 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-12T08:34:03.851 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-12T08:34:03.851 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-12T08:34:03.963 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-827.gd3b5060a.e 1/1 2024-04-12T08:34:03.963 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:03.963 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:03.963 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-immutable-object-cache-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:03.963 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:03.963 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:04.211 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph-mgr 2024-04-12T08:34:04.816 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:03 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:04.885 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-12T08:34:04.885 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:04.885 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:34:04.885 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:04.885 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mds-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:04.885 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:04.885 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-modules-core-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:04.885 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mon-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-osd-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-12T08:34:04.886 INFO:teuthology.orchestra.run.smithi090.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-ply-3.11-14.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-12T08:34:04.887 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:04.888 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:05.101 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:05.101 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:05.102 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:05.103 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.104 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:05.105 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.106 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.107 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.108 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.109 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.110 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:05.110 INFO:teuthology.orchestra.run.smithi114.stdout:Package ceph-mgr-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:05.168 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:05.169 INFO:teuthology.orchestra.run.smithi114.stdout:Nothing to do. 2024-04-12T08:34:05.169 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:05.243 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph-mgr-dashboard 2024-04-12T08:34:05.302 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-base 2024-04-12T08:34:05.831 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:04 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:05.893 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:00:56 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:06.124 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:06.124 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:06.124 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:06.124 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.124 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:06.124 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.124 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:06.124 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:06.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:06.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:06.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:06.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.133 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.133 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.133 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.133 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:06.133 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.133 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.133 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.133 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.180 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:06.180 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:06.180 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:06.180 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.180 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:06.180 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:06.181 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:06.182 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:06.183 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.184 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.185 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.186 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.187 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.188 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.189 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.189 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:06.189 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.189 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.189 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.189 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:06.189 INFO:teuthology.orchestra.run.smithi090.stdout:Package ceph-base-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:06.192 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repository Size 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-dashboard noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 1.7 M 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout:Installing dependencies: 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-grafana-dashboards noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 24 k 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-prometheus-alerts noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 15 k 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout:Installing weak dependencies: 2024-04-12T08:34:06.193 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-12T08:34:06.194 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:06.194 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:06.194 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:06.194 INFO:teuthology.orchestra.run.smithi114.stdout:Install 7 Packages 2024-04-12T08:34:06.194 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:06.194 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 2.0 M 2024-04-12T08:34:06.194 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 8.1 M 2024-04-12T08:34:06.195 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:06.247 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:06.249 INFO:teuthology.orchestra.run.smithi090.stdout:Nothing to do. 2024-04-12T08:34:06.249 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:06.351 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install cephadm 2024-04-12T08:34:06.669 INFO:teuthology.orchestra.run.smithi114.stdout:(1/7): ceph-prometheus-alerts-17.2.7-827.gd3b50 64 kB/s | 15 kB 00:00 2024-04-12T08:34:06.795 INFO:teuthology.orchestra.run.smithi114.stdout:(2/7): ceph-grafana-dashboards-17.2.7-827.gd3b5 69 kB/s | 24 kB 00:00 2024-04-12T08:34:06.845 INFO:teuthology.orchestra.run.smithi114.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 52 kB/s | 9.0 kB 00:00 2024-04-12T08:34:06.936 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:00:57 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:07.062 INFO:teuthology.orchestra.run.smithi114.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 142 kB/s | 31 kB 00:00 2024-04-12T08:34:07.170 INFO:teuthology.orchestra.run.smithi114.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 109 kB/s | 41 kB 00:00 2024-04-12T08:34:07.221 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:07.222 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:07.222 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:07.222 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.222 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:07.222 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.222 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:07.222 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.222 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:07.222 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.222 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.223 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:07.224 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.225 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:07.226 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.227 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.228 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.229 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.229 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.229 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:07.229 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.229 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.229 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.229 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.229 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:07.229 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.230 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.231 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.231 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.231 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:07.231 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.231 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.231 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.231 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:07.288 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:07.289 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:07.289 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-12T08:34:07.289 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:07.289 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:34:07.289 INFO:teuthology.orchestra.run.smithi090.stdout: cephadm noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 82 k 2024-04-12T08:34:07.289 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:07.289 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:34:07.289 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:07.289 INFO:teuthology.orchestra.run.smithi090.stdout:Install 1 Package 2024-04-12T08:34:07.290 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:07.290 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 82 k 2024-04-12T08:34:07.290 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 372 k 2024-04-12T08:34:07.290 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:34:07.312 INFO:teuthology.orchestra.run.smithi114.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 754 kB/s | 188 kB 00:00 2024-04-12T08:34:07.371 INFO:teuthology.orchestra.run.smithi114.stdout:(7/7): ceph-mgr-dashboard-17.2.7-827.gd3b5060a. 1.8 MB/s | 1.7 MB 00:00 2024-04-12T08:34:07.372 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:07.373 INFO:teuthology.orchestra.run.smithi114.stdout:Total 1.7 MB/s | 2.0 MB 00:01 2024-04-12T08:34:07.373 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:07.381 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:07.381 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:07.447 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:07.447 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:07.742 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:07.768 INFO:teuthology.orchestra.run.smithi090.stdout:cephadm-17.2.7-827.gd3b5060a.el9.noarch.rpm 171 kB/s | 82 kB 00:00 2024-04-12T08:34:07.769 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:07.769 INFO:teuthology.orchestra.run.smithi090.stdout:Total 171 kB/s | 82 kB 00:00 2024-04-12T08:34:07.769 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:34:07.774 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:34:07.774 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:34:07.776 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:34:07.776 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:34:07.805 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-12T08:34:07.831 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-12T08:34:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:34:07.898 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-12T08:34:07.926 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-12T08:34:07.955 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-827.gd3b5060a.el9.no 5/7 2024-04-12T08:34:08.178 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-827.gd3b5060a.el9.n 6/7 2024-04-12T08:34:08.208 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-827.gd3b5060a.el9.noarch 7/7 2024-04-12T08:34:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 1/1 2024-04-12T08:34:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 1/1 2024-04-12T08:34:08.719 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-827.gd3b5060a.el9.noarch 7/7 2024-04-12T08:34:08.719 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-827.gd3b5060a.el9.n 1/7 2024-04-12T08:34:08.719 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-827.gd3b5060a.el9.noarch 2/7 2024-04-12T08:34:08.719 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-827.gd3b5060a.el9.no 3/7 2024-04-12T08:34:08.719 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-12T08:34:08.719 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-12T08:34:08.720 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-12T08:34:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 1/1 2024-04-12T08:34:09.064 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-12T08:34:09.064 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:09.064 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:09.064 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-grafana-dashboards-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:09.064 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-dashboard-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:09.064 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-prometheus-alerts-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:09.064 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-12T08:34:09.064 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-12T08:34:09.064 INFO:teuthology.orchestra.run.smithi114.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-12T08:34:09.065 INFO:teuthology.orchestra.run.smithi114.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-12T08:34:09.065 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:09.065 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:09.150 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 1/1 2024-04-12T08:34:09.150 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:09.150 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:34:09.151 INFO:teuthology.orchestra.run.smithi090.stdout: cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:09.151 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:09.151 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:09.289 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-12T08:34:09.348 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-immutable-object-cache 2024-04-12T08:34:09.878 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:08 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:09.953 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:00 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:10.168 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:10.168 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:10.168 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:10.168 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.168 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:10.168 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.168 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:10.168 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.168 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:10.168 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:10.169 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:10.170 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:10.171 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.172 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.173 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.174 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.175 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.176 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.237 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:10.238 INFO:teuthology.orchestra.run.smithi114.stdout:====================================================================================== 2024-04-12T08:34:10.238 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repository Size 2024-04-12T08:34:10.238 INFO:teuthology.orchestra.run.smithi114.stdout:====================================================================================== 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 7.4 M 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout:Installing dependencies: 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-12T08:34:10.239 INFO:teuthology.orchestra.run.smithi114.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-12T08:34:10.240 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:10.240 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:10.240 INFO:teuthology.orchestra.run.smithi114.stdout:====================================================================================== 2024-04-12T08:34:10.240 INFO:teuthology.orchestra.run.smithi114.stdout:Install 12 Packages 2024-04-12T08:34:10.240 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:10.241 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 38 M 2024-04-12T08:34:10.241 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 209 M 2024-04-12T08:34:10.241 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:10.250 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:10.250 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:10.250 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:10.250 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.250 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:10.250 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.250 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.251 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:10.252 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.252 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:10.252 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.252 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:10.252 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.252 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:10.252 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.252 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:10.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:10.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.257 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:10.318 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repo Size 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 137 k 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout:Install 1 Package 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:10.319 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 137 k 2024-04-12T08:34:10.320 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 379 k 2024-04-12T08:34:10.320 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:34:10.605 INFO:teuthology.orchestra.run.smithi090.stdout:ceph-immutable-object-cache-17.2.7-827.gd3b5060 482 kB/s | 137 kB 00:00 2024-04-12T08:34:10.606 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:10.606 INFO:teuthology.orchestra.run.smithi090.stdout:Total 480 kB/s | 137 kB 00:00 2024-04-12T08:34:10.606 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:34:10.613 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:34:10.613 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:34:10.644 INFO:teuthology.orchestra.run.smithi114.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 894 kB/s | 195 kB 00:00 2024-04-12T08:34:10.665 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:34:10.665 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:34:10.778 INFO:teuthology.orchestra.run.smithi114.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 2.2 MB/s | 803 kB 00:00 2024-04-12T08:34:10.802 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:34:10.856 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-827.gd3b5060a.e 1/1 2024-04-12T08:34:10.881 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-827.gd3b5060a.e 1/1 2024-04-12T08:34:10.881 INFO:teuthology.orchestra.run.smithi090.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:34:10.881 INFO:teuthology.orchestra.run.smithi090.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-12T08:34:10.881 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:10.904 INFO:teuthology.orchestra.run.smithi114.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 126 kB/s | 33 kB 00:00 2024-04-12T08:34:10.970 INFO:teuthology.orchestra.run.smithi114.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 277 kB/s | 18 kB 00:00 2024-04-12T08:34:11.029 INFO:teuthology.orchestra.run.smithi114.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 603 kB/s | 35 kB 00:00 2024-04-12T08:34:11.146 INFO:teuthology.orchestra.run.smithi114.stdout:(6/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 8.2 MB/s | 3.0 MB 00:00 2024-04-12T08:34:11.213 INFO:teuthology.orchestra.run.smithi114.stdout:(7/12): python3-devel-3.9.17-2.el9.x86_64.rpm 3.6 MB/s | 245 kB 00:00 2024-04-12T08:34:11.430 INFO:teuthology.orchestra.run.smithi114.stdout:(8/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 24 MB/s | 5.1 MB 00:00 2024-04-12T08:34:11.528 INFO:teuthology.orchestra.run.smithi114.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.5 MB/s | 250 kB 00:00 2024-04-12T08:34:11.689 INFO:teuthology.orchestra.run.smithi114.stdout:(10/12): openblas-openmp-0.3.21-2.el9.x86_64.rp 7.2 MB/s | 4.7 MB 00:00 2024-04-12T08:34:11.797 INFO:teuthology.orchestra.run.smithi114.stdout:(11/12): ceph-mgr-diskprediction-local-17.2.7-8 5.4 MB/s | 7.4 MB 00:01 2024-04-12T08:34:12.015 INFO:teuthology.orchestra.run.smithi114.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-04-12T08:34:12.017 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:12.017 INFO:teuthology.orchestra.run.smithi114.stdout:Total 21 MB/s | 38 MB 00:01 2024-04-12T08:34:12.018 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-827.gd3b5060a.e 1/1 2024-04-12T08:34:12.019 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:12.019 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:34:12.019 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-immutable-object-cache-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:12.019 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:12.019 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:12.351 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-mgr 2024-04-12T08:34:12.367 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:12.394 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:12.394 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:12.900 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:12.901 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:12.957 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:03 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:13.248 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:13.248 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:13.248 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:13.248 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.248 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:13.249 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.249 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:13.249 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.249 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:13.249 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.249 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:13.249 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.249 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:13.249 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.250 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:13.250 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.250 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:13.250 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.250 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:13.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.253 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:13.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.253 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:13.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.253 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:13.253 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.254 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:13.255 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:13.256 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.257 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:13.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.261 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:13.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:13.261 INFO:teuthology.orchestra.run.smithi090.stdout:Package ceph-mgr-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:13.314 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:13.315 INFO:teuthology.orchestra.run.smithi090.stdout:Nothing to do. 2024-04-12T08:34:13.315 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:13.380 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-mgr-dashboard 2024-04-12T08:34:13.655 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:13.778 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-12T08:34:13.820 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-12T08:34:13.919 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-12T08:34:13.955 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-12T08:34:13.969 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:04 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:14.138 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-12T08:34:14.171 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-12T08:34:14.255 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-12T08:34:14.257 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:14.257 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.258 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.259 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:14.260 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.261 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.262 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.263 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.264 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.265 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.266 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:14.327 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:14.327 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:14.328 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-12T08:34:14.328 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:14.328 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:34:14.328 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-dashboard noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 1.7 M 2024-04-12T08:34:14.328 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-12T08:34:14.328 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-grafana-dashboards noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 24 k 2024-04-12T08:34:14.328 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-prometheus-alerts noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 15 k 2024-04-12T08:34:14.328 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-12T08:34:14.328 INFO:teuthology.orchestra.run.smithi090.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-12T08:34:14.328 INFO:teuthology.orchestra.run.smithi090.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-12T08:34:14.329 INFO:teuthology.orchestra.run.smithi090.stdout:Installing weak dependencies: 2024-04-12T08:34:14.329 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-12T08:34:14.329 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:14.329 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:34:14.329 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:14.329 INFO:teuthology.orchestra.run.smithi090.stdout:Install 7 Packages 2024-04-12T08:34:14.329 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:14.330 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 2.0 M 2024-04-12T08:34:14.330 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 8.1 M 2024-04-12T08:34:14.330 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:34:14.900 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-12T08:34:14.914 INFO:teuthology.orchestra.run.smithi090.stdout:(1/7): ceph-grafana-dashboards-17.2.7-827.gd3b5 110 kB/s | 24 kB 00:00 2024-04-12T08:34:14.939 INFO:teuthology.orchestra.run.smithi090.stdout:(2/7): ceph-prometheus-alerts-17.2.7-827.gd3b50 59 kB/s | 15 kB 00:00 2024-04-12T08:34:15.017 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-12T08:34:15.114 INFO:teuthology.orchestra.run.smithi090.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 45 kB/s | 9.0 kB 00:00 2024-04-12T08:34:15.198 INFO:teuthology.orchestra.run.smithi090.stdout:(4/7): ceph-mgr-dashboard-17.2.7-827.gd3b5060a. 3.4 MB/s | 1.7 MB 00:00 2024-04-12T08:34:15.223 INFO:teuthology.orchestra.run.smithi090.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 144 kB/s | 41 kB 00:00 2024-04-12T08:34:15.248 INFO:teuthology.orchestra.run.smithi090.stdout:(6/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 230 kB/s | 31 kB 00:00 2024-04-12T08:34:15.574 INFO:teuthology.orchestra.run.smithi090.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 503 kB/s | 188 kB 00:00 2024-04-12T08:34:15.577 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:15.578 INFO:teuthology.orchestra.run.smithi090.stdout:Total 1.6 MB/s | 2.0 MB 00:01 2024-04-12T08:34:15.578 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:34:15.586 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:34:15.586 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:34:15.649 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:34:15.649 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:34:15.969 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:34:16.034 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-12T08:34:16.067 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-12T08:34:16.123 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-12T08:34:16.154 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-12T08:34:16.193 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-827.gd3b5060a.el9.no 5/7 2024-04-12T08:34:16.311 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-12T08:34:16.421 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-827.gd3b5060a.el9.n 6/7 2024-04-12T08:34:16.455 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-827.gd3b5060a.el9.noarch 7/7 2024-04-12T08:34:17.062 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-827.gd3b5060a.el9.noarch 7/7 2024-04-12T08:34:17.062 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-827.gd3b5060a.el9.n 1/7 2024-04-12T08:34:17.063 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-827.gd3b5060a.el9.noarch 2/7 2024-04-12T08:34:17.063 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-827.gd3b5060a.el9.no 3/7 2024-04-12T08:34:17.063 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-12T08:34:17.063 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-12T08:34:17.063 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-12T08:34:17.334 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-12T08:34:17.371 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-827.gd3b506 12/12 2024-04-12T08:34:17.373 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-12T08:34:17.373 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:17.373 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:34:17.373 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-grafana-dashboards-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:17.374 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-dashboard-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:17.374 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-prometheus-alerts-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:17.374 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-12T08:34:17.374 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-12T08:34:17.374 INFO:teuthology.orchestra.run.smithi090.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-12T08:34:17.374 INFO:teuthology.orchestra.run.smithi090.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-12T08:34:17.374 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:17.374 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:17.645 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-12T08:34:18.273 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:09 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:18.565 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.566 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.567 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:18.568 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.569 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.570 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:18.571 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.572 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.573 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.574 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:18.635 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:18.636 INFO:teuthology.orchestra.run.smithi090.stdout:====================================================================================== 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout:====================================================================================== 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 7.4 M 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-12T08:34:18.637 INFO:teuthology.orchestra.run.smithi090.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-12T08:34:18.638 INFO:teuthology.orchestra.run.smithi090.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-12T08:34:18.638 INFO:teuthology.orchestra.run.smithi090.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-12T08:34:18.638 INFO:teuthology.orchestra.run.smithi090.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-12T08:34:18.638 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:18.638 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:34:18.638 INFO:teuthology.orchestra.run.smithi090.stdout:====================================================================================== 2024-04-12T08:34:18.638 INFO:teuthology.orchestra.run.smithi090.stdout:Install 12 Packages 2024-04-12T08:34:18.638 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:18.639 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 38 M 2024-04-12T08:34:18.639 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 209 M 2024-04-12T08:34:18.639 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:34:19.193 INFO:teuthology.orchestra.run.smithi090.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 648 kB/s | 195 kB 00:00 2024-04-12T08:34:19.252 INFO:teuthology.orchestra.run.smithi090.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 2.2 MB/s | 803 kB 00:00 2024-04-12T08:34:19.277 INFO:teuthology.orchestra.run.smithi090.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 391 kB/s | 33 kB 00:00 2024-04-12T08:34:19.328 INFO:teuthology.orchestra.run.smithi090.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 364 kB/s | 18 kB 00:00 2024-04-12T08:34:19.388 INFO:teuthology.orchestra.run.smithi090.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 592 kB/s | 35 kB 00:00 2024-04-12T08:34:19.488 INFO:teuthology.orchestra.run.smithi090.stdout:(6/12): ceph-mgr-diskprediction-local-17.2.7-82 12 MB/s | 7.4 MB 00:00 2024-04-12T08:34:19.597 INFO:teuthology.orchestra.run.smithi090.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 8.7 MB/s | 3.0 MB 00:00 2024-04-12T08:34:19.647 INFO:teuthology.orchestra.run.smithi090.stdout:(8/12): python3-devel-3.9.17-2.el9.x86_64.rpm 1.5 MB/s | 245 kB 00:00 2024-04-12T08:34:19.781 INFO:teuthology.orchestra.run.smithi090.stdout:(9/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 12 MB/s | 4.7 MB 00:00 2024-04-12T08:34:19.806 INFO:teuthology.orchestra.run.smithi090.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.5 MB/s | 250 kB 00:00 2024-04-12T08:34:19.997 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-827.gd3b506 12/12 2024-04-12T08:34:19.997 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-827.gd3b506 1/12 2024-04-12T08:34:19.997 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-12T08:34:19.997 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-12T08:34:19.997 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-12T08:34:19.997 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-12T08:34:19.997 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-12T08:34:19.998 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-12T08:34:19.998 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-12T08:34:19.998 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 9/12 2024-04-12T08:34:19.998 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-12T08:34:19.998 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-12T08:34:20.133 INFO:teuthology.orchestra.run.smithi090.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 9.5 MB/s | 5.1 MB 00:00 2024-04-12T08:34:20.417 INFO:teuthology.orchestra.run.smithi090.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 26 MB/s | 16 MB 00:00 2024-04-12T08:34:20.418 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:20.418 INFO:teuthology.orchestra.run.smithi090.stdout:Total 21 MB/s | 38 MB 00:01 2024-04-12T08:34:20.434 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-12T08:34:20.434 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:20.434 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:20.434 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-diskprediction-local-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:20.435 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:20.768 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:34:20.794 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:34:20.794 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:34:20.809 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph-mgr-rook 2024-04-12T08:34:21.331 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:34:21.332 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:34:21.413 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:20 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:21.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:21.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.710 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:21.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.710 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.711 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.713 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:21.773 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repository Size 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-rook noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 47 k 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout:Installing dependencies: 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-12T08:34:21.774 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout:Install 13 Packages 2024-04-12T08:34:21.775 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:21.776 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 2.2 M 2024-04-12T08:34:21.776 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 26 M 2024-04-12T08:34:21.776 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:22.083 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:34:22.152 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-12T08:34:22.188 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-12T08:34:22.288 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-12T08:34:22.304 INFO:teuthology.orchestra.run.smithi114.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 244 kB/s | 26 kB 00:00 2024-04-12T08:34:22.330 INFO:teuthology.orchestra.run.smithi114.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 147 kB/s | 19 kB 00:00 2024-04-12T08:34:22.332 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-12T08:34:22.446 INFO:teuthology.orchestra.run.smithi114.stdout:(3/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.3 MB/s | 159 kB 00:00 2024-04-12T08:34:22.472 INFO:teuthology.orchestra.run.smithi114.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.5 MB/s | 222 kB 00:00 2024-04-12T08:34:22.497 INFO:teuthology.orchestra.run.smithi114.stdout:(5/13): ceph-mgr-rook-17.2.7-827.gd3b5060a.el9. 159 kB/s | 47 kB 00:00 2024-04-12T08:34:22.517 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-12T08:34:22.522 INFO:teuthology.orchestra.run.smithi114.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-04-12T08:34:22.547 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-12T08:34:22.556 INFO:teuthology.orchestra.run.smithi114.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 2.5 MB/s | 279 kB 00:00 2024-04-12T08:34:22.641 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-12T08:34:22.789 INFO:teuthology.orchestra.run.smithi114.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 56 kB/s | 15 kB 00:00 2024-04-12T08:34:22.815 INFO:teuthology.orchestra.run.smithi114.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 101 kB/s | 32 kB 00:00 2024-04-12T08:34:22.948 INFO:teuthology.orchestra.run.smithi114.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 443 kB/s | 59 kB 00:00 2024-04-12T08:34:22.999 INFO:teuthology.orchestra.run.smithi114.stdout:(11/13): python3-google-auth-2.28.2-1.el9.noarc 512 kB/s | 203 kB 00:00 2024-04-12T08:34:23.049 INFO:teuthology.orchestra.run.smithi114.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 893 kB/s | 90 kB 00:00 2024-04-12T08:34:23.158 INFO:teuthology.orchestra.run.smithi114.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 2.8 MB/s | 1.0 MB 00:00 2024-04-12T08:34:23.160 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:23.160 INFO:teuthology.orchestra.run.smithi114.stdout:Total 1.6 MB/s | 2.2 MB 00:01 2024-04-12T08:34:23.224 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:23.238 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:23.239 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:23.309 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-12T08:34:23.333 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:23.333 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:23.471 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-12T08:34:23.849 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:24.006 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-12T08:34:24.059 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-12T08:34:24.114 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-12T08:34:24.151 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-12T08:34:24.193 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-12T08:34:24.287 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-12T08:34:24.396 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-12T08:34:24.486 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-12T08:34:24.735 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-12T08:34:25.204 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-12T08:34:25.258 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-12T08:34:25.291 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-12T08:34:25.337 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-12T08:34:25.361 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-mgr-rook-2:17.2.7-827.gd3b5060a.el9.noarch 13/13 2024-04-12T08:34:25.779 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-12T08:34:25.830 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-827.gd3b506 12/12 2024-04-12T08:34:26.191 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-827.gd3b5060a.el9.noarch 13/13 2024-04-12T08:34:26.191 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-mgr-rook-2:17.2.7-827.gd3b5060a.el9.noarch 1/13 2024-04-12T08:34:26.191 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-12T08:34:26.191 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-12T08:34:26.191 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-12T08:34:26.191 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-12T08:34:26.191 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-12T08:34:26.192 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-12T08:34:26.192 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-12T08:34:26.192 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-12T08:34:26.192 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-12T08:34:26.192 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-12T08:34:26.192 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-12T08:34:26.604 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-12T08:34:26.604 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:26.604 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:26.604 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-rook-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:26.604 INFO:teuthology.orchestra.run.smithi114.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-12T08:34:26.604 INFO:teuthology.orchestra.run.smithi114.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-12T08:34:26.604 INFO:teuthology.orchestra.run.smithi114.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-12T08:34:26.604 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-12T08:34:26.604 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-12T08:34:26.605 INFO:teuthology.orchestra.run.smithi114.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-12T08:34:26.605 INFO:teuthology.orchestra.run.smithi114.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-12T08:34:26.605 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-12T08:34:26.605 INFO:teuthology.orchestra.run.smithi114.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-12T08:34:26.605 INFO:teuthology.orchestra.run.smithi114.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-12T08:34:26.605 INFO:teuthology.orchestra.run.smithi114.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-12T08:34:26.605 INFO:teuthology.orchestra.run.smithi114.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-12T08:34:26.605 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:26.605 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:26.829 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph-mgr-cephadm 2024-04-12T08:34:27.424 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:26 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:27.711 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:27.712 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:27.713 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.714 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.715 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.716 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.717 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:27.718 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.719 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.720 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:27.781 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:27.781 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:27.781 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repository Size 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-cephadm noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 123 k 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout:Installing dependencies: 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:27.782 INFO:teuthology.orchestra.run.smithi114.stdout:Install 6 Packages 2024-04-12T08:34:27.783 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:27.783 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 7.0 M 2024-04-12T08:34:27.783 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 33 M 2024-04-12T08:34:27.783 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:28.092 INFO:teuthology.orchestra.run.smithi114.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.5 MB/s | 248 kB 00:00 2024-04-12T08:34:28.200 INFO:teuthology.orchestra.run.smithi114.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 4.9 MB/s | 548 kB 00:00 2024-04-12T08:34:28.234 INFO:teuthology.orchestra.run.smithi114.stdout:(3/6): ceph-mgr-cephadm-17.2.7-827.gd3b5060a.el 404 kB/s | 123 kB 00:00 2024-04-12T08:34:28.259 INFO:teuthology.orchestra.run.smithi114.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 985 kB/s | 58 kB 00:00 2024-04-12T08:34:28.343 INFO:teuthology.orchestra.run.smithi114.stdout:(5/6): python3-babel-2.9.1-2.el9.noarch.rpm 14 MB/s | 6.0 MB 00:00 2024-04-12T08:34:28.368 INFO:teuthology.orchestra.run.smithi114.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 381 kB/s | 51 kB 00:00 2024-04-12T08:34:28.369 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:28.369 INFO:teuthology.orchestra.run.smithi114.stdout:Total 12 MB/s | 7.0 MB 00:00 2024-04-12T08:34:28.442 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:28.454 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:28.454 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:28.557 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:28.557 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:28.876 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:29.016 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-827.gd3b506 12/12 2024-04-12T08:34:29.016 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-827.gd3b506 1/12 2024-04-12T08:34:29.017 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-12T08:34:29.017 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-12T08:34:29.017 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-12T08:34:29.017 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-12T08:34:29.017 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-12T08:34:29.017 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-12T08:34:29.017 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-12T08:34:29.017 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 9/12 2024-04-12T08:34:29.017 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-12T08:34:29.017 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-12T08:34:29.034 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-12T08:34:29.086 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-12T08:34:29.455 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-12T08:34:29.455 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:29.455 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:34:29.455 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-diskprediction-local-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:29.455 INFO:teuthology.orchestra.run.smithi090.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-12T08:34:29.455 INFO:teuthology.orchestra.run.smithi090.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-12T08:34:29.455 INFO:teuthology.orchestra.run.smithi090.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-12T08:34:29.455 INFO:teuthology.orchestra.run.smithi090.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-12T08:34:29.455 INFO:teuthology.orchestra.run.smithi090.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-12T08:34:29.456 INFO:teuthology.orchestra.run.smithi090.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-12T08:34:29.456 INFO:teuthology.orchestra.run.smithi090.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-12T08:34:29.456 INFO:teuthology.orchestra.run.smithi090.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-12T08:34:29.456 INFO:teuthology.orchestra.run.smithi090.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-12T08:34:29.456 INFO:teuthology.orchestra.run.smithi090.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-12T08:34:29.456 INFO:teuthology.orchestra.run.smithi090.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-12T08:34:29.456 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:29.456 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:29.602 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-12T08:34:29.680 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-12T08:34:29.736 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-12T08:34:29.765 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 6/6 2024-04-12T08:34:30.004 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-mgr-rook 2024-04-12T08:34:30.586 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 6/6 2024-04-12T08:34:30.586 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 1/6 2024-04-12T08:34:30.586 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-12T08:34:30.586 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-12T08:34:30.586 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-12T08:34:30.587 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-12T08:34:30.608 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:21 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:30.895 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:30.895 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:30.895 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.896 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:30.897 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.898 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.899 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:30.900 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.901 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:30.902 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.903 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:30.904 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.904 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.904 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.904 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:30.904 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-12T08:34:30.904 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:30.904 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:30.904 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-mgr-cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:30.904 INFO:teuthology.orchestra.run.smithi114.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-12T08:34:30.905 INFO:teuthology.orchestra.run.smithi114.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-12T08:34:30.905 INFO:teuthology.orchestra.run.smithi114.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-12T08:34:30.905 INFO:teuthology.orchestra.run.smithi114.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-12T08:34:30.905 INFO:teuthology.orchestra.run.smithi114.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-12T08:34:30.905 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:30.905 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:30.965 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:30.966 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:30.966 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-12T08:34:30.966 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:30.966 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-rook noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 47 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-12T08:34:30.967 INFO:teuthology.orchestra.run.smithi090.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-12T08:34:30.968 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:30.968 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:34:30.968 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:30.968 INFO:teuthology.orchestra.run.smithi090.stdout:Install 13 Packages 2024-04-12T08:34:30.968 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:30.968 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 2.2 M 2024-04-12T08:34:30.969 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 26 M 2024-04-12T08:34:30.969 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:34:31.137 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install ceph-fuse 2024-04-12T08:34:31.404 INFO:teuthology.orchestra.run.smithi090.stdout:(1/13): ceph-mgr-rook-17.2.7-827.gd3b5060a.el9. 224 kB/s | 47 kB 00:00 2024-04-12T08:34:31.737 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:30 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:31.738 INFO:teuthology.orchestra.run.smithi090.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 35 kB/s | 19 kB 00:00 2024-04-12T08:34:31.871 INFO:teuthology.orchestra.run.smithi090.stdout:(3/13): python3-jsonpatch-1.21-16.el9.noarch.rp 38 kB/s | 26 kB 00:00 2024-04-12T08:34:32.026 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:32.026 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:32.026 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:32.026 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.026 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:32.026 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.026 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:32.027 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:32.028 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.029 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:32.030 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.031 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.032 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.033 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.034 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.035 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:32.097 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:32.097 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:32.097 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repository Size 2024-04-12T08:34:32.097 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:32.097 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:32.097 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-fuse x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 812 k 2024-04-12T08:34:32.097 INFO:teuthology.orchestra.run.smithi114.stdout:Installing dependencies: 2024-04-12T08:34:32.098 INFO:teuthology.orchestra.run.smithi114.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-12T08:34:32.098 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:32.098 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:32.098 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:32.098 INFO:teuthology.orchestra.run.smithi114.stdout:Install 2 Packages 2024-04-12T08:34:32.098 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:32.098 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 892 k 2024-04-12T08:34:32.098 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 2.6 M 2024-04-12T08:34:32.099 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:32.270 INFO:teuthology.orchestra.run.smithi114.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 728 kB/s | 80 kB 00:00 2024-04-12T08:34:32.363 INFO:teuthology.orchestra.run.smithi090.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 232 kB/s | 222 kB 00:00 2024-04-12T08:34:32.428 INFO:teuthology.orchestra.run.smithi114.stdout:(2/2): ceph-fuse-17.2.7-827.gd3b5060a.el9.x86_6 2.9 MB/s | 812 kB 00:00 2024-04-12T08:34:32.429 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:32.429 INFO:teuthology.orchestra.run.smithi114.stdout:Total 2.6 MB/s | 892 kB 00:00 2024-04-12T08:34:32.441 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:32.462 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:32.462 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:32.537 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:32.537 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:32.572 INFO:teuthology.orchestra.run.smithi090.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 190 kB/s | 159 kB 00:00 2024-04-12T08:34:32.731 INFO:teuthology.orchestra.run.smithi090.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 203 kB/s | 32 kB 00:00 2024-04-12T08:34:32.733 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:32.781 INFO:teuthology.orchestra.run.smithi090.stdout:(7/13): python3-certifi-2021.10.8-2.el9.noarch. 298 kB/s | 15 kB 00:00 2024-04-12T08:34:32.806 INFO:teuthology.orchestra.run.smithi090.stdout:(8/13): python3-pyasn1-modules-0.4.8-6.el9.noar 298 kB/s | 279 kB 00:00 2024-04-12T08:34:32.866 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-12T08:34:32.882 INFO:teuthology.orchestra.run.smithi090.stdout:(9/13): python3-google-auth-2.28.2-1.el9.noarch 2.0 MB/s | 203 kB 00:00 2024-04-12T08:34:32.936 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : ceph-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 2/2 2024-04-12T08:34:32.941 INFO:teuthology.orchestra.run.smithi090.stdout:(10/13): python3-kubernetes-26.1.0-2.el9.noarch 7.6 MB/s | 1.0 MB 00:00 2024-04-12T08:34:32.966 INFO:teuthology.orchestra.run.smithi090.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 714 kB/s | 59 kB 00:00 2024-04-12T08:34:32.991 INFO:teuthology.orchestra.run.smithi090.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.8 MB/s | 90 kB 00:00 2024-04-12T08:34:33.033 INFO:teuthology.orchestra.run.smithi090.stdout:(13/13): python3-requests-oauthlib-1.3.0-12.el9 80 kB/s | 54 kB 00:00 2024-04-12T08:34:33.035 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:33.036 INFO:teuthology.orchestra.run.smithi090.stdout:Total 1.1 MB/s | 2.2 MB 00:02 2024-04-12T08:34:33.106 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:34:33.122 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:34:33.122 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:34:33.215 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:34:33.216 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:34:33.637 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: ceph-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 2/2 2024-04-12T08:34:33.637 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : ceph-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 1/2 2024-04-12T08:34:33.726 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:34:33.896 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-12T08:34:33.936 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-12T08:34:33.937 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:33.937 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:33.937 INFO:teuthology.orchestra.run.smithi114.stdout: ceph-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-12T08:34:33.937 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:33.937 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:33.943 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-12T08:34:33.999 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-12T08:34:34.037 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-12T08:34:34.078 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-12T08:34:34.172 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-12T08:34:34.213 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install librados-devel 2024-04-12T08:34:34.290 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-12T08:34:34.348 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-12T08:34:34.826 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:33 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:35.062 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-12T08:34:35.114 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:35.115 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:35.116 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:35.117 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.118 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.119 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.120 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:35.121 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.122 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:35.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.123 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:35.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:35.175 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-12T08:34:35.185 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:35.185 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:35.185 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repo Size 2024-04-12T08:34:35.185 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:35.185 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:35.186 INFO:teuthology.orchestra.run.smithi114.stdout: librados-devel x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 126 k 2024-04-12T08:34:35.186 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:35.186 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:35.186 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:35.186 INFO:teuthology.orchestra.run.smithi114.stdout:Install 1 Package 2024-04-12T08:34:35.186 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:35.186 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 126 k 2024-04-12T08:34:35.187 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 448 k 2024-04-12T08:34:35.187 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:35.206 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-12T08:34:35.251 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-12T08:34:35.275 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-rook-2:17.2.7-827.gd3b5060a.el9.noarch 13/13 2024-04-12T08:34:35.416 INFO:teuthology.orchestra.run.smithi114.stdout:librados-devel-17.2.7-827.gd3b5060a.el9.x86_64. 550 kB/s | 126 kB 00:00 2024-04-12T08:34:35.417 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:35.417 INFO:teuthology.orchestra.run.smithi114.stdout:Total 547 kB/s | 126 kB 00:00 2024-04-12T08:34:35.417 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:35.423 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:35.423 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:35.481 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:35.482 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:35.630 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:35.739 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : librados-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:36.210 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: librados-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:36.249 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-827.gd3b5060a.el9.noarch 13/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-rook-2:17.2.7-827.gd3b5060a.el9.noarch 1/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-12T08:34:36.250 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-12T08:34:36.519 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : librados-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:36.519 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:36.519 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:36.520 INFO:teuthology.orchestra.run.smithi114.stdout: librados-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:36.520 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:36.520 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-rook-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-12T08:34:36.623 INFO:teuthology.orchestra.run.smithi090.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-12T08:34:36.624 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-12T08:34:36.624 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-12T08:34:36.624 INFO:teuthology.orchestra.run.smithi090.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-12T08:34:36.624 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-12T08:34:36.624 INFO:teuthology.orchestra.run.smithi090.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-12T08:34:36.624 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:36.624 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:36.782 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install libcephfs2 2024-04-12T08:34:36.912 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-mgr-cephadm 2024-04-12T08:34:37.385 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:36 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:37.507 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:28 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:37.680 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:37.681 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.682 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:37.683 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:37.684 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.685 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.686 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.687 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.688 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.689 INFO:teuthology.orchestra.run.smithi114.stdout:Package libcephfs2-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:37.748 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:37.749 INFO:teuthology.orchestra.run.smithi114.stdout:Nothing to do. 2024-04-12T08:34:37.750 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:37.795 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.796 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:37.797 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.798 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.799 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:37.800 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.801 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.802 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.803 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:37.804 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.804 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.804 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.804 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.804 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:37.804 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.804 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.804 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.804 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:37.823 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install libcephfs-devel 2024-04-12T08:34:37.866 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:37.867 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:37.867 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-12T08:34:37.867 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:37.867 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:34:37.867 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-cephadm noarch 2:17.2.7-827.gd3b5060a.el9 ceph-noarch 123 k 2024-04-12T08:34:37.867 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-12T08:34:37.867 INFO:teuthology.orchestra.run.smithi090.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-12T08:34:37.867 INFO:teuthology.orchestra.run.smithi090.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-12T08:34:37.867 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-12T08:34:37.867 INFO:teuthology.orchestra.run.smithi090.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-12T08:34:37.868 INFO:teuthology.orchestra.run.smithi090.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-12T08:34:37.868 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:37.868 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:34:37.868 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:37.868 INFO:teuthology.orchestra.run.smithi090.stdout:Install 6 Packages 2024-04-12T08:34:37.868 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:37.868 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 7.0 M 2024-04-12T08:34:37.868 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 33 M 2024-04-12T08:34:37.869 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:34:38.318 INFO:teuthology.orchestra.run.smithi090.stdout:(1/6): ceph-mgr-cephadm-17.2.7-827.gd3b5060a.el 526 kB/s | 123 kB 00:00 2024-04-12T08:34:38.414 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:37 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:38.677 INFO:teuthology.orchestra.run.smithi090.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.7 MB/s | 548 kB 00:00 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:38.701 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.702 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.703 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.704 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.705 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:38.706 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.707 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.708 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.709 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:38.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.709 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:38.735 INFO:teuthology.orchestra.run.smithi090.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 989 kB/s | 58 kB 00:00 2024-04-12T08:34:38.769 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:38.770 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:38.770 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repo Size 2024-04-12T08:34:38.770 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:38.770 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:38.770 INFO:teuthology.orchestra.run.smithi114.stdout: libcephfs-devel x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 25 k 2024-04-12T08:34:38.770 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:38.770 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:38.770 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:38.770 INFO:teuthology.orchestra.run.smithi114.stdout:Install 1 Package 2024-04-12T08:34:38.770 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:38.771 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 25 k 2024-04-12T08:34:38.771 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 110 k 2024-04-12T08:34:38.771 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:38.802 INFO:teuthology.orchestra.run.smithi090.stdout:(4/6): python3-typing-extensions-4.4.0-2.el9.no 762 kB/s | 51 kB 00:00 2024-04-12T08:34:39.037 INFO:teuthology.orchestra.run.smithi114.stdout:libcephfs-devel-17.2.7-827.gd3b5060a.el9.x86_64 94 kB/s | 25 kB 00:00 2024-04-12T08:34:39.037 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:39.037 INFO:teuthology.orchestra.run.smithi114.stdout:Total 94 kB/s | 25 kB 00:00 2024-04-12T08:34:39.037 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:39.040 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:39.040 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:39.042 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:39.042 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:39.053 INFO:teuthology.orchestra.run.smithi090.stdout:(5/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 256 kB/s | 248 kB 00:00 2024-04-12T08:34:39.122 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:39.233 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : libcephfs-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:39.624 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:39.902 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : libcephfs-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:39.902 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:39.902 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:39.902 INFO:teuthology.orchestra.run.smithi114.stdout: libcephfs-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:39.902 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:39.902 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:40.075 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install librados2 2024-04-12T08:34:40.668 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:39 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:40.957 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:40.957 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:40.957 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:40.957 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:40.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:40.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:40.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:40.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.964 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.966 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:40.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:40.966 INFO:teuthology.orchestra.run.smithi114.stdout:Package librados2-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:41.027 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:41.028 INFO:teuthology.orchestra.run.smithi114.stdout:Nothing to do. 2024-04-12T08:34:41.028 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:41.102 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install librbd1 2024-04-12T08:34:41.696 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:40 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:41.984 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:41.984 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:41.984 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:41.984 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.984 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:41.985 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:41.986 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.987 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:41.988 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.989 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.990 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.991 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.992 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.993 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:41.993 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.993 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.993 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.993 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:41.993 INFO:teuthology.orchestra.run.smithi114.stdout:Package librbd1-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:42.053 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:42.054 INFO:teuthology.orchestra.run.smithi114.stdout:Nothing to do. 2024-04-12T08:34:42.054 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:42.122 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install python3-rados 2024-04-12T08:34:42.718 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:41 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:43.012 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:43.012 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:43.012 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.013 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.014 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:43.015 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.016 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:43.016 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.016 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:43.016 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.016 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:43.016 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.016 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:43.016 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.016 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:43.016 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:43.017 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.018 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.019 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.020 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.021 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:43.022 INFO:teuthology.orchestra.run.smithi114.stdout:Package python3-rados-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:43.081 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:43.082 INFO:teuthology.orchestra.run.smithi114.stdout:Nothing to do. 2024-04-12T08:34:43.083 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:43.152 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install python3-rgw 2024-04-12T08:34:43.746 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:42 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:44.036 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:44.037 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.038 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:44.039 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:44.040 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:44.041 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:44.042 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.043 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:44.044 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.045 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.046 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.047 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:44.047 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.047 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.047 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.047 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:44.047 INFO:teuthology.orchestra.run.smithi114.stdout:Package python3-rgw-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:44.105 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:44.106 INFO:teuthology.orchestra.run.smithi114.stdout:Nothing to do. 2024-04-12T08:34:44.106 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:44.185 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install python3-cephfs 2024-04-12T08:34:44.780 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:43 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:45.066 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:45.066 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:45.066 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:45.066 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.066 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:45.066 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.066 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:45.066 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.067 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:45.068 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:45.069 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.070 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.071 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.072 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.073 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.074 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:45.075 INFO:teuthology.orchestra.run.smithi114.stdout:Package python3-cephfs-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:45.157 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:45.158 INFO:teuthology.orchestra.run.smithi114.stdout:Nothing to do. 2024-04-12T08:34:45.158 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:45.239 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install python3-rbd 2024-04-12T08:34:45.831 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:44 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:46.123 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:46.123 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:46.123 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:46.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.123 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:46.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.123 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:46.123 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.123 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:46.124 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.125 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:46.126 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:46.127 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:46.128 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.129 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.130 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:46.131 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.132 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:46.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.132 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:46.132 INFO:teuthology.orchestra.run.smithi114.stdout:Package python3-rbd-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:46.192 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:46.193 INFO:teuthology.orchestra.run.smithi114.stdout:Nothing to do. 2024-04-12T08:34:46.193 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:46.261 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install rbd-fuse 2024-04-12T08:34:46.661 INFO:teuthology.orchestra.run.smithi090.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 712 kB/s | 6.0 MB 00:08 2024-04-12T08:34:46.662 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:46.663 INFO:teuthology.orchestra.run.smithi090.stdout:Total 811 kB/s | 7.0 MB 00:08 2024-04-12T08:34:46.736 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:34:46.747 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:34:46.747 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:34:46.849 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:34:46.849 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:34:46.866 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:45 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:47.156 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:47.156 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:47.156 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:47.156 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.156 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:47.156 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.156 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:47.157 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:47.158 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:47.159 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:47.160 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:47.161 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.162 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:47.163 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.164 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.165 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:47.165 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.165 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.165 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.165 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:47.165 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:34:47.226 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:47.226 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:47.226 INFO:teuthology.orchestra.run.smithi114.stdout: Package Architecture Version Repository Size 2024-04-12T08:34:47.226 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:47.226 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:47.226 INFO:teuthology.orchestra.run.smithi114.stdout: rbd-fuse x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 82 k 2024-04-12T08:34:47.227 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:47.227 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:47.227 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:47.227 INFO:teuthology.orchestra.run.smithi114.stdout:Install 1 Package 2024-04-12T08:34:47.227 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:47.227 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 82 k 2024-04-12T08:34:47.227 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 222 k 2024-04-12T08:34:47.227 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:47.325 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-12T08:34:47.380 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-12T08:34:47.891 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-12T08:34:48.000 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-12T08:34:48.064 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-12T08:34:48.094 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 6/6 2024-04-12T08:34:48.747 INFO:teuthology.orchestra.run.smithi114.stdout:rbd-fuse-17.2.7-827.gd3b5060a.el9.x86_64.rpm 54 kB/s | 82 kB 00:01 2024-04-12T08:34:48.748 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:48.748 INFO:teuthology.orchestra.run.smithi114.stdout:Total 54 kB/s | 82 kB 00:01 2024-04-12T08:34:48.749 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:48.757 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:48.757 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:48.820 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:48.820 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:48.960 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:49.061 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : rbd-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:49.189 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 6/6 2024-04-12T08:34:49.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 1/6 2024-04-12T08:34:49.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-12T08:34:49.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-12T08:34:49.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-12T08:34:49.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-12T08:34:49.486 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: rbd-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:49.516 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-12T08:34:49.516 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:49.516 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:34:49.516 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-cephadm-2:17.2.7-827.gd3b5060a.el9.noarch 2024-04-12T08:34:49.516 INFO:teuthology.orchestra.run.smithi090.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-12T08:34:49.516 INFO:teuthology.orchestra.run.smithi090.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-12T08:34:49.516 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-12T08:34:49.516 INFO:teuthology.orchestra.run.smithi090.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-12T08:34:49.517 INFO:teuthology.orchestra.run.smithi090.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-12T08:34:49.517 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:49.517 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:49.797 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : rbd-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:49.797 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:49.797 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:49.797 INFO:teuthology.orchestra.run.smithi114.stdout: rbd-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:49.797 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:49.798 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:49.812 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-fuse 2024-04-12T08:34:50.041 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install rbd-mirror 2024-04-12T08:34:50.410 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:41 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:50.665 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:49 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:50.700 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:50.700 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:50.700 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:50.700 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.701 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:50.702 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.703 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.704 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.705 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:50.706 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.707 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:50.708 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.709 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.709 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.709 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.709 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:50.709 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.709 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.709 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.709 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.770 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:50.770 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:50.770 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-12T08:34:50.771 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:50.771 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:34:50.771 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-fuse x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 812 k 2024-04-12T08:34:50.771 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-12T08:34:50.771 INFO:teuthology.orchestra.run.smithi090.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-12T08:34:50.771 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:50.771 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:34:50.771 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:50.771 INFO:teuthology.orchestra.run.smithi090.stdout:Install 2 Packages 2024-04-12T08:34:50.771 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:50.772 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 892 k 2024-04-12T08:34:50.772 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 2.6 M 2024-04-12T08:34:50.772 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:34:50.957 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:50.957 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.958 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.959 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.960 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.961 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.962 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:50.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.963 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:50.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.963 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:50.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.963 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:50.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.963 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.964 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.965 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.966 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:50.967 INFO:teuthology.orchestra.run.smithi090.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 628 kB/s | 80 kB 00:00 2024-04-12T08:34:51.026 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:51.027 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:51.027 INFO:teuthology.orchestra.run.smithi114.stdout: Package Arch Version Repo Size 2024-04-12T08:34:51.027 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:51.027 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:51.027 INFO:teuthology.orchestra.run.smithi114.stdout: rbd-mirror x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 2.9 M 2024-04-12T08:34:51.027 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:51.027 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:51.027 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:51.027 INFO:teuthology.orchestra.run.smithi114.stdout:Install 1 Package 2024-04-12T08:34:51.027 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:51.028 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 2.9 M 2024-04-12T08:34:51.028 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 12 M 2024-04-12T08:34:51.028 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:51.285 INFO:teuthology.orchestra.run.smithi090.stdout:(2/2): ceph-fuse-17.2.7-827.gd3b5060a.el9.x86_6 1.8 MB/s | 812 kB 00:00 2024-04-12T08:34:51.285 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:51.285 INFO:teuthology.orchestra.run.smithi090.stdout:Total 1.7 MB/s | 892 kB 00:00 2024-04-12T08:34:51.297 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:34:51.319 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:34:51.319 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:34:51.394 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:34:51.395 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:34:51.400 INFO:teuthology.orchestra.run.smithi114.stdout:rbd-mirror-17.2.7-827.gd3b5060a.el9.x86_64.rpm 7.9 MB/s | 2.9 MB 00:00 2024-04-12T08:34:51.401 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:51.401 INFO:teuthology.orchestra.run.smithi114.stdout:Total 7.9 MB/s | 2.9 MB 00:00 2024-04-12T08:34:51.401 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:51.410 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:51.410 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:51.486 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:51.487 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:51.580 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:34:51.714 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-12T08:34:51.785 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 2/2 2024-04-12T08:34:51.910 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:51.959 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : rbd-mirror-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:51.985 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: rbd-mirror-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:51.985 INFO:teuthology.orchestra.run.smithi114.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:34:51.985 INFO:teuthology.orchestra.run.smithi114.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-12T08:34:51.985 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-12T08:34:51.985 INFO:teuthology.orchestra.run.smithi114.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-12T08:34:51.985 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:52.519 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 2/2 2024-04-12T08:34:52.519 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 1/2 2024-04-12T08:34:52.839 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-12T08:34:52.839 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:52.839 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:34:52.839 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-12T08:34:52.840 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:52.840 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:53.127 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : rbd-mirror-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:53.128 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:53.128 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:53.128 INFO:teuthology.orchestra.run.smithi114.stdout: rbd-mirror-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:53.128 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:53.128 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:53.264 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install librados-devel 2024-04-12T08:34:53.380 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum -y install rbd-nbd 2024-04-12T08:34:53.869 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:44 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:53.994 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:01:52 ago on Fri 12 Apr 2024 08:33:01 AM UTC. 2024-04-12T08:34:54.161 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:54.161 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:54.161 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.162 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.163 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.164 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.165 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.166 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:54.167 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.168 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.169 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.170 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.170 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:54.170 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.170 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.170 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.170 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.231 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:54.231 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:54.231 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repo Size 2024-04-12T08:34:54.231 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:54.231 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:34:54.231 INFO:teuthology.orchestra.run.smithi090.stdout: librados-devel x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 126 k 2024-04-12T08:34:54.231 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:54.232 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:34:54.232 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:54.232 INFO:teuthology.orchestra.run.smithi090.stdout:Install 1 Package 2024-04-12T08:34:54.232 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:54.232 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 126 k 2024-04-12T08:34:54.232 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 448 k 2024-04-12T08:34:54.233 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:34:54.283 INFO:teuthology.orchestra.run.smithi114.stderr:Modular dependency problems: 2024-04-12T08:34:54.283 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T08:34:54.283 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 1: conflicting requests 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 2: conflicting requests 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 3: conflicting requests 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 4: conflicting requests 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 5: conflicting requests 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 6: conflicting requests 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 7: conflicting requests 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.284 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 8: conflicting requests 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 9: conflicting requests 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 10: conflicting requests 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 11: conflicting requests 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 12: conflicting requests 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 13: conflicting requests 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 14: conflicting requests 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.285 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 15: conflicting requests 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 16: conflicting requests 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 17: conflicting requests 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 18: conflicting requests 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 19: conflicting requests 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 20: conflicting requests 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 21: conflicting requests 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.286 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 22: conflicting requests 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 23: conflicting requests 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 24: conflicting requests 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 25: conflicting requests 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 26: conflicting requests 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 27: conflicting requests 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 28: conflicting requests 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.287 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 29: conflicting requests 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 30: conflicting requests 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 31: conflicting requests 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 32: conflicting requests 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 33: conflicting requests 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 34: conflicting requests 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 35: conflicting requests 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 36: conflicting requests 2024-04-12T08:34:54.288 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 37: conflicting requests 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 39: conflicting requests 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.289 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:54.290 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.291 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.292 INFO:teuthology.orchestra.run.smithi114.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:54.292 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.292 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.292 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.292 INFO:teuthology.orchestra.run.smithi114.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:54.353 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-12T08:34:54.354 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:54.354 INFO:teuthology.orchestra.run.smithi114.stdout: Package Architecture Version Repository Size 2024-04-12T08:34:54.354 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:54.354 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-04-12T08:34:54.354 INFO:teuthology.orchestra.run.smithi114.stdout: rbd-nbd x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 165 k 2024-04-12T08:34:54.354 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:54.354 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-12T08:34:54.354 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-12T08:34:54.354 INFO:teuthology.orchestra.run.smithi114.stdout:Install 1 Package 2024-04-12T08:34:54.354 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:54.355 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 165 k 2024-04-12T08:34:54.355 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 477 k 2024-04-12T08:34:54.355 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-12T08:34:54.495 INFO:teuthology.orchestra.run.smithi090.stdout:librados-devel-17.2.7-827.gd3b5060a.el9.x86_64. 481 kB/s | 126 kB 00:00 2024-04-12T08:34:54.496 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:54.496 INFO:teuthology.orchestra.run.smithi090.stdout:Total 479 kB/s | 126 kB 00:00 2024-04-12T08:34:54.496 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:34:54.501 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:34:54.501 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:34:54.559 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:34:54.560 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:34:54.587 INFO:teuthology.orchestra.run.smithi114.stdout:rbd-nbd-17.2.7-827.gd3b5060a.el9.x86_64.rpm 710 kB/s | 165 kB 00:00 2024-04-12T08:34:54.588 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:54.588 INFO:teuthology.orchestra.run.smithi114.stdout:Total 705 kB/s | 165 kB 00:00 2024-04-12T08:34:54.588 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-12T08:34:54.595 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-12T08:34:54.596 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-12T08:34:54.655 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-12T08:34:54.655 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-12T08:34:54.704 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:34:54.794 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : librados-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:54.806 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-12T08:34:54.918 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : rbd-nbd-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:55.347 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librados-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:55.365 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: rbd-nbd-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:55.659 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librados-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:55.659 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:55.659 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:34:55.659 INFO:teuthology.orchestra.run.smithi090.stdout: librados-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:55.659 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:55.659 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:55.699 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : rbd-nbd-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:55.699 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:55.700 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-04-12T08:34:55.700 INFO:teuthology.orchestra.run.smithi114.stdout: rbd-nbd-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:55.700 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:34:55.700 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-12T08:34:55.933 DEBUG:teuthology.parallel:result is None 2024-04-12T08:34:55.944 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install libcephfs2 2024-04-12T08:34:56.551 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:47 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:56.842 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:56.843 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:56.844 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:56.845 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:56.846 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.847 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.848 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.849 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:56.850 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.851 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.851 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.851 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.851 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:56.851 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.851 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.851 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.851 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:56.852 INFO:teuthology.orchestra.run.smithi090.stdout:Package libcephfs2-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:34:56.911 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:56.912 INFO:teuthology.orchestra.run.smithi090.stdout:Nothing to do. 2024-04-12T08:34:56.912 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:56.989 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install libcephfs-devel 2024-04-12T08:34:57.583 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:48 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:34:57.871 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:34:57.872 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:34:57.873 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.874 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.875 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.876 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:34:57.877 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.878 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:34:57.879 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.879 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.879 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.879 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:34:57.941 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:34:57.941 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:57.941 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repo Size 2024-04-12T08:34:57.941 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:57.941 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:34:57.941 INFO:teuthology.orchestra.run.smithi090.stdout: libcephfs-devel x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 25 k 2024-04-12T08:34:57.941 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:57.942 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:34:57.942 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:34:57.942 INFO:teuthology.orchestra.run.smithi090.stdout:Install 1 Package 2024-04-12T08:34:57.942 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:57.942 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 25 k 2024-04-12T08:34:57.942 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 110 k 2024-04-12T08:34:57.942 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:34:58.261 INFO:teuthology.orchestra.run.smithi090.stdout:libcephfs-devel-17.2.7-827.gd3b5060a.el9.x86_64 78 kB/s | 25 kB 00:00 2024-04-12T08:34:58.261 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:34:58.262 INFO:teuthology.orchestra.run.smithi090.stdout:Total 78 kB/s | 25 kB 00:00 2024-04-12T08:34:58.262 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:34:58.265 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:34:58.265 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:34:58.266 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:34:58.267 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:34:58.345 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:34:58.459 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libcephfs-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:58.900 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:59.182 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libcephfs-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:34:59.182 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:59.182 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:34:59.182 INFO:teuthology.orchestra.run.smithi090.stdout: libcephfs-devel-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:34:59.182 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:34:59.182 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:34:59.383 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install librados2 2024-04-12T08:34:59.990 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:50 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:35:00.279 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:35:00.279 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:35:00.279 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:35:00.279 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.279 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:35:00.279 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.279 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:35:00.279 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:35:00.280 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:35:00.281 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.282 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.283 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.284 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.285 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.286 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.287 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.290 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:35:00.290 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.290 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.290 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.290 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:35:00.291 INFO:teuthology.orchestra.run.smithi090.stdout:Package librados2-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:35:00.349 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:35:00.350 INFO:teuthology.orchestra.run.smithi090.stdout:Nothing to do. 2024-04-12T08:35:00.350 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:35:00.451 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install librbd1 2024-04-12T08:35:01.053 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:52 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:35:01.342 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:35:01.342 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:35:01.342 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:35:01.342 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.342 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:35:01.342 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.342 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.343 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.344 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:35:01.345 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:35:01.346 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.347 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:35:01.348 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.349 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.350 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.351 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.351 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:35:01.351 INFO:teuthology.orchestra.run.smithi090.stdout:Package librbd1-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:35:01.412 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:35:01.413 INFO:teuthology.orchestra.run.smithi090.stdout:Nothing to do. 2024-04-12T08:35:01.413 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:35:01.497 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install python3-rados 2024-04-12T08:35:02.090 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:53 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:35:02.378 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:35:02.378 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.379 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.380 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:35:02.381 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.381 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:35:02.381 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.381 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:35:02.381 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.381 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:35:02.381 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.381 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:35:02.381 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.381 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.382 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:35:02.383 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.384 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.385 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.386 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:35:02.387 INFO:teuthology.orchestra.run.smithi090.stdout:Package python3-rados-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:35:02.446 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:35:02.447 INFO:teuthology.orchestra.run.smithi090.stdout:Nothing to do. 2024-04-12T08:35:02.447 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:35:02.547 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install python3-rgw 2024-04-12T08:35:03.156 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:54 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:35:03.444 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:35:03.444 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:35:03.444 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:35:03.445 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.446 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.447 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:35:03.448 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.449 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.450 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.451 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:35:03.452 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.453 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.453 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.453 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.453 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:35:03.453 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.453 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.453 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.453 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:35:03.453 INFO:teuthology.orchestra.run.smithi090.stdout:Package python3-rgw-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:35:03.515 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:35:03.516 INFO:teuthology.orchestra.run.smithi090.stdout:Nothing to do. 2024-04-12T08:35:03.516 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:35:03.615 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install python3-cephfs 2024-04-12T08:35:04.216 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:55 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:35:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:35:04.510 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:35:04.510 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:35:04.510 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.510 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:35:04.510 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.510 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:35:04.510 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.510 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:35:04.511 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:35:04.512 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.513 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.514 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.515 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.516 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.517 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.518 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:35:04.518 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.518 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.518 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.518 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:35:04.518 INFO:teuthology.orchestra.run.smithi090.stdout:Package python3-cephfs-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:35:04.581 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:35:04.582 INFO:teuthology.orchestra.run.smithi090.stdout:Nothing to do. 2024-04-12T08:35:04.583 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:35:04.697 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install python3-rbd 2024-04-12T08:35:05.289 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:56 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:35:05.575 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:35:05.575 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:35:05.575 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:35:05.575 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.575 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:35:05.576 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:35:05.577 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:35:05.578 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:35:05.579 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:35:05.580 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.581 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:35:05.582 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.583 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.584 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:35:05.584 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.584 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.584 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.584 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:35:05.584 INFO:teuthology.orchestra.run.smithi090.stdout:Package python3-rbd-2:17.2.7-827.gd3b5060a.el9.x86_64 is already installed. 2024-04-12T08:35:05.642 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:35:05.643 INFO:teuthology.orchestra.run.smithi090.stdout:Nothing to do. 2024-04-12T08:35:05.644 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:35:05.717 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install rbd-fuse 2024-04-12T08:35:06.320 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:57 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:35:06.609 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:35:06.609 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:35:06.609 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:35:06.609 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.609 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:35:06.609 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.609 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:35:06.609 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.609 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:35:06.610 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.611 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:35:06.612 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.613 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.614 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.615 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.616 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.617 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.618 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.618 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:35:06.618 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.618 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.618 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.618 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:35:06.679 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:35:06.679 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout: Package Architecture Version Repository Size 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-fuse x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 82 k 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout:Install 1 Package 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 82 k 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 222 k 2024-04-12T08:35:06.680 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:35:06.884 INFO:teuthology.orchestra.run.smithi090.stdout:rbd-fuse-17.2.7-827.gd3b5060a.el9.x86_64.rpm 402 kB/s | 82 kB 00:00 2024-04-12T08:35:06.884 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:35:06.884 INFO:teuthology.orchestra.run.smithi090.stdout:Total 400 kB/s | 82 kB 00:00 2024-04-12T08:35:06.885 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:35:06.891 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:35:06.891 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:35:06.950 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:35:06.950 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:35:07.089 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:35:07.189 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : rbd-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:35:07.658 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: rbd-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:35:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : rbd-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:35:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:07.998 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:35:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-fuse-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:35:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:07.998 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:35:08.279 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install rbd-mirror 2024-04-12T08:35:08.878 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:01:59 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:35:09.170 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:35:09.171 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:35:09.172 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.173 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:35:09.174 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:35:09.175 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:35:09.176 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.177 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.178 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.179 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:35:09.239 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:35:09.239 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repo Size 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-mirror x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 2.9 M 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout:Install 1 Package 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 2.9 M 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 12 M 2024-04-12T08:35:09.240 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:35:09.976 INFO:teuthology.orchestra.run.smithi090.stdout:rbd-mirror-17.2.7-827.gd3b5060a.el9.x86_64.rpm 4.0 MB/s | 2.9 MB 00:00 2024-04-12T08:35:09.976 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:35:09.976 INFO:teuthology.orchestra.run.smithi090.stdout:Total 4.0 MB/s | 2.9 MB 00:00 2024-04-12T08:35:09.977 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:35:09.986 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:35:09.986 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:35:10.061 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:35:10.061 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:35:10.470 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:35:10.531 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : rbd-mirror-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:35:10.551 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: rbd-mirror-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:35:10.551 INFO:teuthology.orchestra.run.smithi090.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T08:35:10.551 INFO:teuthology.orchestra.run.smithi090.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-12T08:35:10.551 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-12T08:35:10.551 INFO:teuthology.orchestra.run.smithi090.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-12T08:35:10.551 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:11.842 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : rbd-mirror-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:35:11.842 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:11.842 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:35:11.842 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-mirror-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:35:11.842 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:11.842 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:35:12.123 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install rbd-nbd 2024-04-12T08:35:12.788 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:02:03 ago on Fri 12 Apr 2024 08:33:09 AM UTC. 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr:Modular dependency problems: 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 1: conflicting requests 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 2: conflicting requests 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 3: conflicting requests 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 4: conflicting requests 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.082 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 5: conflicting requests 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 6: conflicting requests 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 7: conflicting requests 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 8: conflicting requests 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 9: conflicting requests 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 10: conflicting requests 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 11: conflicting requests 2024-04-12T08:35:13.083 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 12: conflicting requests 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 13: conflicting requests 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 14: conflicting requests 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 15: conflicting requests 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 16: conflicting requests 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 17: conflicting requests 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 18: conflicting requests 2024-04-12T08:35:13.084 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 19: conflicting requests 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 20: conflicting requests 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 21: conflicting requests 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 22: conflicting requests 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 23: conflicting requests 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 24: conflicting requests 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 25: conflicting requests 2024-04-12T08:35:13.085 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 26: conflicting requests 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 27: conflicting requests 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 28: conflicting requests 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 29: conflicting requests 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 30: conflicting requests 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.086 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 31: conflicting requests 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 32: conflicting requests 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 33: conflicting requests 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 34: conflicting requests 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 35: conflicting requests 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 36: conflicting requests 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 37: conflicting requests 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.087 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 39: conflicting requests 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.088 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T08:35:13.089 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.090 INFO:teuthology.orchestra.run.smithi090.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T08:35:13.151 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-12T08:35:13.151 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:35:13.151 INFO:teuthology.orchestra.run.smithi090.stdout: Package Architecture Version Repository Size 2024-04-12T08:35:13.151 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:35:13.151 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-12T08:35:13.151 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-nbd x86_64 2:17.2.7-827.gd3b5060a.el9 ceph 165 k 2024-04-12T08:35:13.151 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:13.151 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-12T08:35:13.152 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-12T08:35:13.152 INFO:teuthology.orchestra.run.smithi090.stdout:Install 1 Package 2024-04-12T08:35:13.152 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:13.152 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 165 k 2024-04-12T08:35:13.152 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 477 k 2024-04-12T08:35:13.152 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-12T08:35:13.493 INFO:teuthology.orchestra.run.smithi090.stdout:rbd-nbd-17.2.7-827.gd3b5060a.el9.x86_64.rpm 485 kB/s | 165 kB 00:00 2024-04-12T08:35:13.493 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:35:13.493 INFO:teuthology.orchestra.run.smithi090.stdout:Total 483 kB/s | 165 kB 00:00 2024-04-12T08:35:13.494 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-12T08:35:13.501 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-12T08:35:13.501 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-12T08:35:13.559 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-12T08:35:13.560 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-12T08:35:13.707 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-12T08:35:13.807 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : rbd-nbd-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:35:14.296 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: rbd-nbd-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:35:14.606 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : rbd-nbd-2:17.2.7-827.gd3b5060a.el9.x86_64 1/1 2024-04-12T08:35:14.606 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:14.606 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-12T08:35:14.606 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-nbd-2:17.2.7-827.gd3b5060a.el9.x86_64 2024-04-12T08:35:14.606 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:35:14.606 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-12T08:35:14.871 DEBUG:teuthology.parallel:result is None 2024-04-12T08:35:14.871 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T08:35:14.871 INFO:teuthology.packaging:ref: None 2024-04-12T08:35:14.871 INFO:teuthology.packaging:tag: None 2024-04-12T08:35:14.871 INFO:teuthology.packaging:branch: quincy 2024-04-12T08:35:14.871 INFO:teuthology.packaging:sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:35:14.872 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-12T08:35:15.212 DEBUG:teuthology.orchestra.run.smithi090:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-12T08:35:15.234 INFO:teuthology.orchestra.run.smithi090.stdout:17.2.7-827.gd3b5060a.el9 2024-04-12T08:35:15.235 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-827.gd3b5060a.el9 2024-04-12T08:35:15.235 INFO:teuthology.task.install:The correct ceph version 17.2.7-827.gd3b5060a is installed. 2024-04-12T08:35:15.236 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T08:35:15.236 INFO:teuthology.packaging:ref: None 2024-04-12T08:35:15.237 INFO:teuthology.packaging:tag: None 2024-04-12T08:35:15.237 INFO:teuthology.packaging:branch: quincy 2024-04-12T08:35:15.237 INFO:teuthology.packaging:sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:35:15.237 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-12T08:35:15.466 DEBUG:teuthology.orchestra.run.smithi114:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-12T08:35:15.490 INFO:teuthology.orchestra.run.smithi114.stdout:17.2.7-827.gd3b5060a.el9 2024-04-12T08:35:15.491 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-827.gd3b5060a.el9 2024-04-12T08:35:15.491 INFO:teuthology.task.install:The correct ceph version 17.2.7-827.gd3b5060a is installed. 2024-04-12T08:35:15.493 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-12T08:35:15.493 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:35:15.493 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-12T08:35:15.521 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:35:15.521 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-12T08:35:15.564 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-12T08:35:15.564 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:35:15.564 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/usr/bin/daemon-helper 2024-04-12T08:35:15.591 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-12T08:35:15.657 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:35:15.657 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/usr/bin/daemon-helper 2024-04-12T08:35:15.688 DEBUG:teuthology.orchestra.run.smithi114:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-12T08:35:15.759 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-12T08:35:15.759 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:35:15.759 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-12T08:35:15.786 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-12T08:35:15.853 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:35:15.853 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-12T08:35:15.884 DEBUG:teuthology.orchestra.run.smithi114:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-12T08:35:15.954 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-12T08:35:15.954 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:35:15.954 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/usr/bin/stdin-killer 2024-04-12T08:35:15.981 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-12T08:35:16.048 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:35:16.048 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/usr/bin/stdin-killer 2024-04-12T08:35:16.077 DEBUG:teuthology.orchestra.run.smithi114:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-12T08:35:16.146 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-12T08:35:16.242 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b277feb2c7beb4597d8ecc39d58d5c36b81f613a'} 2024-04-12T08:35:16.242 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-04-12T08:35:16.243 INFO:tasks.cephadm:Cluster fsid is 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:35:16.243 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-12T08:35:16.243 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.90', 'mon.c': '[v2:172.21.15.90:3301,v1:172.21.15.90:6790]', 'mon.b': '172.21.15.114'} 2024-04-12T08:35:16.243 INFO:tasks.cephadm:First mon is mon.a on smithi090 2024-04-12T08:35:16.243 INFO:tasks.cephadm:First mgr is y 2024-04-12T08:35:16.243 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-12T08:35:16.244 DEBUG:teuthology.orchestra.run.smithi090:> sudo hostname $(hostname -s) 2024-04-12T08:35:16.271 DEBUG:teuthology.orchestra.run.smithi114:> sudo hostname $(hostname -s) 2024-04-12T08:35:16.301 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-04-12T08:35:16.301 DEBUG:teuthology.orchestra.run.smithi090:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-12T08:35:16.479 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 12 08:35 /home/ubuntu/cephtest/cephadm 2024-04-12T08:35:16.479 DEBUG:teuthology.orchestra.run.smithi114:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-12T08:35:16.650 INFO:teuthology.orchestra.run.smithi114.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 12 08:35 /home/ubuntu/cephtest/cephadm 2024-04-12T08:35:16.651 DEBUG:teuthology.orchestra.run.smithi090:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-12T08:35:16.667 DEBUG:teuthology.orchestra.run.smithi114:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-12T08:35:16.694 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-04-12T08:35:16.694 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-12T08:35:16.709 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-12T08:35:17.620 INFO:teuthology.orchestra.run.smithi114.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-12T08:35:17.726 INFO:teuthology.orchestra.run.smithi090.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-12T08:36:14.885 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-12T08:36:14.886 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph_version": "ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)", 2024-04-12T08:36:14.886 INFO:teuthology.orchestra.run.smithi090.stdout: "image_id": "21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241", 2024-04-12T08:36:14.886 INFO:teuthology.orchestra.run.smithi090.stdout: "repo_digests": [ 2024-04-12T08:36:14.886 INFO:teuthology.orchestra.run.smithi090.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe" 2024-04-12T08:36:14.886 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-12T08:36:14.886 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-12T08:36:15.109 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-04-12T08:36:15.109 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph_version": "ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)", 2024-04-12T08:36:15.109 INFO:teuthology.orchestra.run.smithi114.stdout: "image_id": "21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241", 2024-04-12T08:36:15.109 INFO:teuthology.orchestra.run.smithi114.stdout: "repo_digests": [ 2024-04-12T08:36:15.109 INFO:teuthology.orchestra.run.smithi114.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe" 2024-04-12T08:36:15.109 INFO:teuthology.orchestra.run.smithi114.stdout: ] 2024-04-12T08:36:15.109 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-04-12T08:36:15.129 DEBUG:teuthology.orchestra.run.smithi090:> sudo mkdir -p /etc/ceph 2024-04-12T08:36:15.159 DEBUG:teuthology.orchestra.run.smithi114:> sudo mkdir -p /etc/ceph 2024-04-12T08:36:15.188 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod 777 /etc/ceph 2024-04-12T08:36:15.225 DEBUG:teuthology.orchestra.run.smithi114:> sudo chmod 777 /etc/ceph 2024-04-12T08:36:15.255 INFO:tasks.cephadm:Writing seed config... 2024-04-12T08:36:15.256 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-12T08:36:15.256 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-12T08:36:15.256 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-12T08:36:15.256 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-12T08:36:15.256 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-12T08:36:15.256 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-12T08:36:15.257 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-12T08:36:15.257 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-12T08:36:15.257 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-12T08:36:15.257 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-12T08:36:15.257 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-12T08:36:15.257 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:36:15.257 DEBUG:teuthology.orchestra.run.smithi090:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-12T08:36:15.280 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 96735376-f8a7-11ee-bc8c-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-12T08:36:15.280 DEBUG:teuthology.orchestra.run.smithi090:mon.a> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.a.service 2024-04-12T08:36:15.323 DEBUG:teuthology.orchestra.run.smithi090:mgr.y> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.y.service 2024-04-12T08:36:15.365 INFO:tasks.cephadm:Bootstrapping... 2024-04-12T08:36:15.366 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.90 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-12T08:36:15.570 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-12T08:36:15.570 INFO:teuthology.orchestra.run.smithi090.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '96735376-f8a7-11ee-bc8c-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.90', '--skip-admin-label'] 2024-04-12T08:36:15.597 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-12T08:36:15.597 INFO:teuthology.orchestra.run.smithi090.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-12T08:36:15.597 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying podman|docker is present... 2024-04-12T08:36:15.621 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-12T08:36:15.622 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying lvm2 is present... 2024-04-12T08:36:15.622 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying time synchronization is in place... 2024-04-12T08:36:15.629 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-12T08:36:15.629 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-12T08:36:15.636 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-12T08:36:15.636 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-12T08:36:15.642 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout enabled 2024-04-12T08:36:15.649 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout active 2024-04-12T08:36:15.649 INFO:teuthology.orchestra.run.smithi090.stdout:Unit chronyd.service is enabled and running 2024-04-12T08:36:15.649 INFO:teuthology.orchestra.run.smithi090.stdout:Repeating the final host check... 2024-04-12T08:36:15.673 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-12T08:36:15.673 INFO:teuthology.orchestra.run.smithi090.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-12T08:36:15.673 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl is present 2024-04-12T08:36:15.674 INFO:teuthology.orchestra.run.smithi090.stdout:lvcreate is present 2024-04-12T08:36:15.680 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-12T08:36:15.680 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-12T08:36:15.685 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-12T08:36:15.685 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-12T08:36:15.692 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout enabled 2024-04-12T08:36:15.698 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout active 2024-04-12T08:36:15.698 INFO:teuthology.orchestra.run.smithi090.stdout:Unit chronyd.service is enabled and running 2024-04-12T08:36:15.698 INFO:teuthology.orchestra.run.smithi090.stdout:Host looks OK 2024-04-12T08:36:15.699 INFO:teuthology.orchestra.run.smithi090.stdout:Cluster fsid: 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:36:15.699 INFO:teuthology.orchestra.run.smithi090.stdout:Acquiring lock 140197967909984 on /run/cephadm/96735376-f8a7-11ee-bc8c-c7b262605968.lock 2024-04-12T08:36:15.699 INFO:teuthology.orchestra.run.smithi090.stdout:Lock 140197967909984 acquired on /run/cephadm/96735376-f8a7-11ee-bc8c-c7b262605968.lock 2024-04-12T08:36:15.699 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying IP 172.21.15.90 port 3300 ... 2024-04-12T08:36:15.699 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying IP 172.21.15.90 port 6789 ... 2024-04-12T08:36:15.699 INFO:teuthology.orchestra.run.smithi090.stdout:Base mon IP(s) is [172.21.15.90:3300, 172.21.15.90:6789], mon addrv is [v2:172.21.15.90:3300,v1:172.21.15.90:6789] 2024-04-12T08:36:15.702 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.90 metric 100 2024-04-12T08:36:15.703 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.90 metric 100 2024-04-12T08:36:15.705 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-12T08:36:15.705 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-12T08:36:15.705 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1669sec hoplimit 64 pref medium 2024-04-12T08:36:15.708 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-12T08:36:15.708 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-12T08:36:15.708 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-12T08:36:15.708 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-12T08:36:15.708 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d767/64 scope link 2024-04-12T08:36:15.708 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-12T08:36:15.709 INFO:teuthology.orchestra.run.smithi090.stdout:Mon IP `172.21.15.90` is in CIDR network `172.21.0.0/20` 2024-04-12T08:36:15.709 INFO:teuthology.orchestra.run.smithi090.stdout:Mon IP `172.21.15.90` is in CIDR network `172.21.0.0/20` 2024-04-12T08:36:15.709 INFO:teuthology.orchestra.run.smithi090.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-12T08:36:15.709 INFO:teuthology.orchestra.run.smithi090.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-12T08:36:15.710 INFO:teuthology.orchestra.run.smithi090.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-12T08:36:16.196 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stdout 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 2024-04-12T08:36:16.196 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-12T08:36:16.196 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Getting image source signatures 2024-04-12T08:36:16.196 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Copying blob sha256:6a38c9a31ea4c6c1e3d079c8edc2f3b418b00f3ce1d3a48ffe92dcb2ef7b4e51 2024-04-12T08:36:16.196 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Copying blob sha256:9793b5b5582798bde394f4383d142e9f248a25e4ebfd72ee05ea9ec6251a877d 2024-04-12T08:36:16.196 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Copying config sha256:21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 2024-04-12T08:36:16.196 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-12T08:36:17.094 INFO:teuthology.orchestra.run.smithi090.stdout:ceph: stdout ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable) 2024-04-12T08:36:17.094 INFO:teuthology.orchestra.run.smithi090.stdout:Ceph version: ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable) 2024-04-12T08:36:17.094 INFO:teuthology.orchestra.run.smithi090.stdout:Extracting ceph user uid/gid from container image... 2024-04-12T08:36:17.976 INFO:teuthology.orchestra.run.smithi090.stdout:stat: stdout 167 167 2024-04-12T08:36:17.976 INFO:teuthology.orchestra.run.smithi090.stdout:Creating initial keys... 2024-04-12T08:36:18.859 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph-authtool: stdout AQCC8hhmwoJrERAAB08Vc4KpaHn7KfXRa/1rhA== 2024-04-12T08:36:27.970 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph-authtool: stdout AQCD8hhm28BgIRAAnvMXaQP2IS80aFlUvZHcUw== 2024-04-12T08:36:31.569 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph-authtool: stdout AQCO8hhmZqZNOhAAEpael3wnfy2MSyB+Mbh0MQ== 2024-04-12T08:36:31.570 INFO:teuthology.orchestra.run.smithi090.stdout:Creating initial monmap... 2024-04-12T08:36:32.453 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-12T08:36:32.453 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-04-12T08:36:32.453 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:36:32.453 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-12T08:36:32.453 INFO:teuthology.orchestra.run.smithi090.stdout:monmaptool for a [v2:172.21.15.90:3300,v1:172.21.15.90:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-12T08:36:32.453 INFO:teuthology.orchestra.run.smithi090.stdout:setting min_mon_release = octopus 2024-04-12T08:36:32.453 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: set fsid to 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:36:32.453 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-12T08:36:32.453 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:36:32.453 INFO:teuthology.orchestra.run.smithi090.stdout:Creating mon... 2024-04-12T08:36:33.822 INFO:teuthology.orchestra.run.smithi090.stdout:create mon.a on 2024-04-12T08:36:34.143 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-12T08:36:34.362 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-12T08:36:34.514 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-96735376-f8a7-11ee-bc8c-c7b262605968.target → /etc/systemd/system/ceph-96735376-f8a7-11ee-bc8c-c7b262605968.target. 2024-04-12T08:36:34.514 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-96735376-f8a7-11ee-bc8c-c7b262605968.target → /etc/systemd/system/ceph-96735376-f8a7-11ee-bc8c-c7b262605968.target. 2024-04-12T08:36:34.710 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.a 2024-04-12T08:36:34.711 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Failed to reset failed state of unit ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.a.service: Unit ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.a.service not loaded. 2024-04-12T08:36:34.883 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-96735376-f8a7-11ee-bc8c-c7b262605968.target.wants/ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.a.service → /etc/systemd/system/ceph-96735376-f8a7-11ee-bc8c-c7b262605968@.service. 2024-04-12T08:36:35.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:35 smithi090 podman[28711]: 2024-04-12 08:36:35.031746635 +0000 UTC m=+0.018072220 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-12T08:36:35.438 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-12T08:36:35.439 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout disabled 2024-04-12T08:36:35.445 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-12T08:36:35.445 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-12T08:36:35.445 INFO:teuthology.orchestra.run.smithi090.stdout:firewalld.service is not enabled 2024-04-12T08:36:35.446 INFO:teuthology.orchestra.run.smithi090.stdout:Not possible to enable service . firewalld.service is not available 2024-04-12T08:36:35.446 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mon to start... 2024-04-12T08:36:35.446 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mon... 2024-04-12T08:36:35.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:35 smithi090 podman[28711]: 2024-04-12 08:36:35.331593037 +0000 UTC m=+0.317918627 container init 433db03a2f9eef6b7d6b3d5cbe6725a8196672deb5d7f13106beb9481e386e27 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , RELEASE=quincy-d3b5060) 2024-04-12T08:36:35.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:35 smithi090 podman[28711]: 2024-04-12 08:36:35.334934979 +0000 UTC m=+0.321260559 container start 433db03a2f9eef6b7d6b3d5cbe6725a8196672deb5d7f13106beb9481e386e27 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, RELEASE=quincy-d3b5060, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-12T08:36:35.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:35 smithi090 ceph-mon[28729]: mkfs 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:36:35.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:35 smithi090 ceph-mon[28729]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-12T08:36:35.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:35 smithi090 bash[28711]: 433db03a2f9eef6b7d6b3d5cbe6725a8196672deb5d7f13106beb9481e386e27 2024-04-12T08:36:35.496 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:35 smithi090 systemd[1]: Started Ceph mon.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:36:36.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:36 smithi090 ceph-mon[28729]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-12T08:36:36.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:36 smithi090 ceph-mon[28729]: monmap e1: 1 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T08:36:36.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:36 smithi090 ceph-mon[28729]: fsmap 2024-04-12T08:36:36.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:36 smithi090 ceph-mon[28729]: osdmap e1: 0 total, 0 up, 0 in 2024-04-12T08:36:36.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:36 smithi090 ceph-mon[28729]: mgrmap e1: no daemons active 2024-04-12T08:36:37.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:37 smithi090 ceph-mon[28729]: from='client.? 172.21.15.90:0/209470830' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-12T08:36:37.721 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout cluster: 2024-04-12T08:36:37.721 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout id: 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:36:37.721 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-12T08:36:37.721 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:37.721 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout services: 2024-04-12T08:36:37.721 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 1.77618s) 2024-04-12T08:36:37.721 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-12T08:36:37.721 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-12T08:36:37.721 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:37.721 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout data: 2024-04-12T08:36:37.722 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-12T08:36:37.722 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-12T08:36:37.722 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-12T08:36:37.722 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout pgs: 2024-04-12T08:36:37.722 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:37.722 INFO:teuthology.orchestra.run.smithi090.stdout:mon is available 2024-04-12T08:36:37.722 INFO:teuthology.orchestra.run.smithi090.stdout:Assimilating anything we can from ceph.conf... 2024-04-12T08:36:38.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:38 smithi090 ceph-mon[28729]: from='client.? 172.21.15.90:0/3797873475' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-12T08:36:38.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:38 smithi090 ceph-mon[28729]: from='client.? 172.21.15.90:0/3797873475' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-12T08:36:38.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:38.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [global] 2024-04-12T08:36:38.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout fsid = 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:36:38.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.90:3300,v1:172.21.15.90:6789] 2024-04-12T08:36:38.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-12T08:36:38.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-12T08:36:38.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-12T08:36:38.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-12T08:36:38.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:38.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-12T08:36:38.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-12T08:36:38.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:38.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [osd] 2024-04-12T08:36:38.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-12T08:36:38.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-12T08:36:38.756 INFO:teuthology.orchestra.run.smithi090.stdout:Generating new minimal ceph.conf... 2024-04-12T08:36:39.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:39 smithi090 ceph-mon[28729]: from='client.? 172.21.15.90:0/4209712432' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:36:39.737 INFO:teuthology.orchestra.run.smithi090.stdout:Restarting the monitor... 2024-04-12T08:36:39.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:39 smithi090 systemd[1]: Stopping Ceph mon.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:36:40.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:39 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a[28725]: 2024-04-12T08:36:39.995+0000 7f708a2cc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T08:36:40.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:39 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a[28725]: 2024-04-12T08:36:39.995+0000 7f708a2cc700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-12T08:36:40.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:40 smithi090 podman[28974]: 2024-04-12 08:36:40.190164723 +0000 UTC m=+0.299853524 container died 433db03a2f9eef6b7d6b3d5cbe6725a8196672deb5d7f13106beb9481e386e27 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-12T08:36:40.706 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:40 smithi090 podman[28974]: 2024-04-12 08:36:40.50435843 +0000 UTC m=+0.614047223 container cleanup 433db03a2f9eef6b7d6b3d5cbe6725a8196672deb5d7f13106beb9481e386e27 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-12T08:36:40.706 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:40 smithi090 bash[28974]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a 2024-04-12T08:36:41.056 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:40 smithi090 podman[28988]: 2024-04-12 08:36:40.705863367 +0000 UTC m=+0.510938406 container remove 433db03a2f9eef6b7d6b3d5cbe6725a8196672deb5d7f13106beb9481e386e27 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-12T08:36:41.056 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:40 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-12T08:36:41.056 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:40 smithi090 systemd[1]: Stopped Ceph mon.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:36:41.057 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:40 smithi090 systemd[1]: Starting Ceph mon.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:36:41.298 INFO:teuthology.orchestra.run.smithi090.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 podman[29082]: 2024-04-12 08:36:40.959323492 +0000 UTC m=+0.017557142 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 podman[29082]: 2024-04-12 08:36:41.056837679 +0000 UTC m=+0.115071330 container create 5c90fd1861be8cb014276016d32819c28aa4c3b8c34bdd98f0a066b4905146d6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 podman[29082]: 2024-04-12 08:36:41.235760019 +0000 UTC m=+0.293993669 container init 5c90fd1861be8cb014276016d32819c28aa4c3b8c34bdd98f0a066b4905146d6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=quincy-d3b5060) 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 podman[29082]: 2024-04-12 08:36:41.238612728 +0000 UTC m=+0.296846379 container start 5c90fd1861be8cb014276016d32819c28aa4c3b8c34bdd98f0a066b4905146d6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux ) 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable), process ceph-mon, pid 2 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: pidfile_write: ignore empty --pid-file 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: load: jerasure load: lrc 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: RocksDB version: 6.15.5 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Compile date Apr 9 2024 2024-04-12T08:36:41.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: DB SUMMARY 2024-04-12T08:36:41.328 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: DB Session ID: PQLJ0CD6AFXBQIXF4665 2024-04-12T08:36:41.328 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: CURRENT file: CURRENT 2024-04-12T08:36:41.328 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T08:36:41.328 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-12T08:36:41.328 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-12T08:36:41.328 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84510 ; 2024-04-12T08:36:41.328 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.error_if_exists: 0 2024-04-12T08:36:41.328 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.create_if_missing: 0 2024-04-12T08:36:41.328 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.paranoid_checks: 1 2024-04-12T08:36:41.329 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T08:36:41.329 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.env: 0x5645ccf58a80 2024-04-12T08:36:41.329 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.fs: Posix File System 2024-04-12T08:36:41.329 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.info_log: 0x5645cef3ce80 2024-04-12T08:36:41.329 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T08:36:41.329 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.statistics: (nil) 2024-04-12T08:36:41.329 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.use_fsync: 0 2024-04-12T08:36:41.329 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_log_file_size: 0 2024-04-12T08:36:41.329 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.allow_fallocate: 1 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.use_direct_reads: 0 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.db_log_dir: 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T08:36:41.330 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.write_buffer_manager: 0x5645cfcac960 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T08:36:41.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.unordered_write: 0 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.row_cache: None 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.wal_filter: None 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.preserve_deletes: 0 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.two_write_queues: 0 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.atomic_flush: 0 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T08:36:41.332 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.log_readahead_size: 0 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_background_jobs: 2 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_background_compactions: -1 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_subcompactions: 1 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T08:36:41.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_open_files: -1 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_background_flushes: -1 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Compression algorithms supported: 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: kZSTD supported: 0 2024-04-12T08:36:41.334 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: kXpressCompression supported: 0 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: kZlibCompression supported: 1 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: kSnappyCompression supported: 1 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: kLZ4Compression supported: 1 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: kBZip2Compression supported: 0 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.merge_operator: 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_filter: None 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T08:36:41.335 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5645cef8cf58) 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: cache_index_and_filter_blocks: 1 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: pin_top_level_index_and_filter: 1 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: index_type: 0 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: data_block_index_type: 0 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: index_shortening: 1 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: hash_index_allow_collision: 1 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: checksum: 1 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: no_block_cache: 0 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache: 0x5645cefbf090 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache_name: BinnedLRUCache 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache_options: 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: capacity : 536870912 2024-04-12T08:36:41.336 INFO:journalctl@ceph.mon.a.smithi090.stdout: num_shard_bits : 4 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: strict_capacity_limit : 0 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: high_pri_pool_ratio: 0.000 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache_compressed: (nil) 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: persistent_cache: (nil) 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_size: 4096 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_size_deviation: 10 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_restart_interval: 16 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: index_block_restart_interval: 1 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: metadata_block_size: 4096 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: partition_filters: 0 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: use_delta_encoding: 1 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-12T08:36:41.337 INFO:journalctl@ceph.mon.a.smithi090.stdout: whole_key_filtering: 1 2024-04-12T08:36:41.338 INFO:journalctl@ceph.mon.a.smithi090.stdout: verify_compression: 0 2024-04-12T08:36:41.338 INFO:journalctl@ceph.mon.a.smithi090.stdout: read_amp_bytes_per_bit: 0 2024-04-12T08:36:41.338 INFO:journalctl@ceph.mon.a.smithi090.stdout: format_version: 4 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout: enable_index_compression: 1 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_align: 0 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compression: NoCompression 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.num_levels: 7 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T08:36:41.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T08:36:41.340 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T08:36:41.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.arena_block_size: 4194304 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T08:36:41.342 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.table_properties_collectors: 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.inplace_update_support: 0 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.bloom_locality: 0 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.max_successive_merges: 0 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.ttl: 2592000 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.enable_blob_files: false 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.min_blob_size: 0 2024-04-12T08:36:41.343 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911001264265, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911001264955, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80794, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 79055, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10297, "raw_average_key_size": 47, "raw_value_size": 73238, "raw_average_value_size": 337, "num_data_blocks": 10, "num_entries": 217, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712911001, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "0ab2179f-5b2c-421a-bf41-73f53e994c06", "db_session_id": "PQLJ0CD6AFXBQIXF4665"}} 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911001265311, "job": 1, "event": "recovery_finished"} 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T08:36:41.344 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5645cef53180 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: DB pointer 0x5645cf028000 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** DB Stats ** 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:36:41.345 INFO:journalctl@ceph.mon.a.smithi090.stdout: L0 2/0 80.66 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 169.3 0.00 0.00 1 0.000 0 0 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: Sum 2/0 80.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 169.3 0.00 0.00 1 0.000 0 0 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 169.3 0.00 0.00 1 0.000 0 0 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 169.3 0.00 0.00 1 0.000 0 0 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative compaction: 0.00 GB write, 26.92 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:36:41.346 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval compaction: 0.00 GB write, 26.92 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: L0 2/0 80.66 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 169.3 0.00 0.00 1 0.000 0 0 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: Sum 2/0 80.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 169.3 0.00 0.00 1 0.000 0 0 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:36:41.347 INFO:journalctl@ceph.mon.a.smithi090.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 169.3 0.00 0.00 1 0.000 0 0 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative compaction: 0.00 GB write, 26.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: starting mon.a rank 0 at public addrs [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] at bind addrs [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: mon.a@-1(???) e1 preinit fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: mon.a@-1(???).mds e1 new map 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: mon.a@-1(???).mds e1 print_map 2024-04-12T08:36:41.348 INFO:journalctl@ceph.mon.a.smithi090.stdout: e1 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout: legacy client fscid: -1 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout: No filesystems configured 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 bash[29082]: 5c90fd1861be8cb014276016d32819c28aa4c3b8c34bdd98f0a066b4905146d6 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 systemd[1]: Started Ceph mon.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: monmap e1: 1 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T08:36:41.349 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: fsmap 2024-04-12T08:36:41.350 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: osdmap e1: 0 total, 0 up, 0 in 2024-04-12T08:36:41.350 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:41 smithi090 ceph-mon[29100]: mgrmap e1: no daemons active 2024-04-12T08:36:42.616 INFO:teuthology.orchestra.run.smithi090.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-12T08:36:42.616 INFO:teuthology.orchestra.run.smithi090.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-12T08:36:42.616 INFO:teuthology.orchestra.run.smithi090.stdout:Creating mgr... 2024-04-12T08:36:42.616 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying port 9283 ... 2024-04-12T08:36:42.792 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.y 2024-04-12T08:36:42.792 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Failed to reset failed state of unit ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.y.service: Unit ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.y.service not loaded. 2024-04-12T08:36:42.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:42 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2227420296' entity='client.admin' 2024-04-12T08:36:42.945 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-96735376-f8a7-11ee-bc8c-c7b262605968.target.wants/ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-96735376-f8a7-11ee-bc8c-c7b262605968@.service. 2024-04-12T08:36:43.458 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-12T08:36:43.458 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout disabled 2024-04-12T08:36:43.466 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-12T08:36:43.466 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-12T08:36:43.466 INFO:teuthology.orchestra.run.smithi090.stdout:firewalld.service is not enabled 2024-04-12T08:36:43.467 INFO:teuthology.orchestra.run.smithi090.stdout:Not possible to enable service . firewalld.service is not available 2024-04-12T08:36:43.477 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-12T08:36:43.477 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout disabled 2024-04-12T08:36:43.485 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-12T08:36:43.486 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-12T08:36:43.486 INFO:teuthology.orchestra.run.smithi090.stdout:firewalld.service is not enabled 2024-04-12T08:36:43.486 INFO:teuthology.orchestra.run.smithi090.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-12T08:36:43.486 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mgr to start... 2024-04-12T08:36:43.486 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mgr... 2024-04-12T08:36:43.601 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:43 smithi090 systemd[1]: Started Ceph mgr.y for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:36:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:44 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/456546511' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T08:36:44.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:43.878+0000 7f054ffee1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:36:44.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:43.983+0000 7f054ffee1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:36:44.562 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:44.562 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-12T08:36:44.562 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsid": "96735376-f8a7-11ee-bc8c-c7b262605968", 2024-04-12T08:36:44.562 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "health": { 2024-04-12T08:36:44.562 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 0 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "a" 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:44.563 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:44.564 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:44.565 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "restful" 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modified": "2024-04-12T08:36:35.371448+0000", 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:44.566 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-12T08:36:44.567 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-12T08:36:44.567 INFO:teuthology.orchestra.run.smithi090.stdout:mgr not available, waiting (1/15)... 2024-04-12T08:36:44.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:44.620+0000 7f054ffee1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:36:44.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:44.707+0000 7f054ffee1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:36:44.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:44.789+0000 7f054ffee1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:36:45.365 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:44.936+0000 7f054ffee1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:36:45.365 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:45.019+0000 7f054ffee1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:36:45.365 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:45.186+0000 7f054ffee1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:36:45.365 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:45.363+0000 7f054ffee1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:36:45.626 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:45.446+0000 7f054ffee1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:36:45.627 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:45.537+0000 7f054ffee1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:36:45.627 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:45.625+0000 7f054ffee1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:36:46.840 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:46.677+0000 7f054ffee1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:36:46.840 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:46.814+0000 7f054ffee1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:36:47.120 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:47.047+0000 7f054ffee1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:36:47.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:47 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2889968659' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T08:36:47.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:47.281+0000 7f054ffee1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:36:47.743 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:47.456+0000 7f054ffee1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:36:47.744 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:47.638+0000 7f054ffee1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:36:47.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:47.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-12T08:36:47.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsid": "96735376-f8a7-11ee-bc8c-c7b262605968", 2024-04-12T08:36:47.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "health": { 2024-04-12T08:36:47.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-12T08:36:47.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-12T08:36:47.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 0 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "a" 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:47.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-12T08:36:47.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-12T08:36:47.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "restful" 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modified": "2024-04-12T08:36:35.371448+0000", 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-12T08:36:47.756 INFO:teuthology.orchestra.run.smithi090.stdout:mgr not available, waiting (2/15)... 2024-04-12T08:36:48.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:47.900+0000 7f054ffee1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:36:48.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:47.982+0000 7f054ffee1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:36:48.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:48 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:48.359+0000 7f054ffee1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:36:49.393 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:49.110+0000 7f054ffee1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:36:49.393 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:49.392+0000 7f054ffee1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:36:50.087 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:49.698+0000 7f054ffee1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:36:50.087 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:49.891+0000 7f054ffee1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:36:50.087 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: Activating manager daemon y 2024-04-12T08:36:50.087 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: mgrmap e2: y(active, starting, since 0.0028864s) 2024-04-12T08:36:50.087 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: from='mgr.14100 172.21.15.90:0/2374578661' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:36:50.087 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: from='mgr.14100 172.21.15.90:0/2374578661' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:36:50.087 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: from='mgr.14100 172.21.15.90:0/2374578661' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:36:50.088 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: from='mgr.14100 172.21.15.90:0/2374578661' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:36:50.088 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: from='mgr.14100 172.21.15.90:0/2374578661' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:36:50.088 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: Manager daemon y is now available 2024-04-12T08:36:50.088 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: from='mgr.14100 172.21.15.90:0/2374578661' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:36:50.088 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: from='mgr.14100 172.21.15.90:0/2374578661' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:36:50.088 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: from='mgr.14100 172.21.15.90:0/2374578661' entity='mgr.y' 2024-04-12T08:36:50.088 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: from='mgr.14100 172.21.15.90:0/2374578661' entity='mgr.y' 2024-04-12T08:36:50.088 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:49 smithi090 ceph-mon[29100]: from='mgr.14100 172.21.15.90:0/2374578661' entity='mgr.y' 2024-04-12T08:36:51.053 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:50 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3448459826' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T08:36:51.053 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:50 smithi090 ceph-mon[29100]: mgrmap e3: y(active, since 1.00634s) 2024-04-12T08:36:51.056 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:51.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-12T08:36:51.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsid": "96735376-f8a7-11ee-bc8c-c7b262605968", 2024-04-12T08:36:51.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "health": { 2024-04-12T08:36:51.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-12T08:36:51.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-12T08:36:51.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-12T08:36:51.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:51.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-12T08:36:51.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-12T08:36:51.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 0 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "a" 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-12T08:36:51.061 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-12T08:36:51.062 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-12T08:36:51.062 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-12T08:36:51.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-12T08:36:51.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-12T08:36:51.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:51.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-12T08:36:51.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-12T08:36:51.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-12T08:36:51.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "restful" 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modified": "2024-04-12T08:36:35.371448+0000", 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-12T08:36:51.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-12T08:36:51.068 INFO:teuthology.orchestra.run.smithi090.stdout:mgr not available, waiting (3/15)... 2024-04-12T08:36:53.285 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:52 smithi090 ceph-mon[29100]: mgrmap e4: y(active, since 2s) 2024-04-12T08:36:54.103 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:53 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2651257858' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T08:36:54.392 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:54.392 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-12T08:36:54.392 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsid": "96735376-f8a7-11ee-bc8c-c7b262605968", 2024-04-12T08:36:54.392 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "health": { 2024-04-12T08:36:54.392 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-12T08:36:54.392 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 0 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "a" 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:54.393 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-12T08:36:54.394 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:54.394 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-12T08:36:54.394 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-12T08:36:54.394 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-12T08:36:54.397 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-12T08:36:54.397 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-12T08:36:54.397 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:54.398 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "restful" 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-12T08:36:54.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T08:36:54.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:54.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-12T08:36:54.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T08:36:54.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modified": "2024-04-12T08:36:35.371448+0000", 2024-04-12T08:36:54.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T08:36:54.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-12T08:36:54.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-12T08:36:54.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-12T08:36:54.400 INFO:teuthology.orchestra.run.smithi090.stdout:mgr is available 2024-04-12T08:36:55.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:55 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2183516619' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-12T08:36:55.583 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:55.583 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [global] 2024-04-12T08:36:55.583 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout fsid = 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [osd] 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-12T08:36:55.584 INFO:teuthology.orchestra.run.smithi090.stdout:Enabling cephadm module... 2024-04-12T08:36:56.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:56 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4260041679' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-12T08:36:56.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ignoring --setuser ceph since I am not root 2024-04-12T08:36:56.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ignoring --setgroup ceph since I am not root 2024-04-12T08:36:56.975 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:56.812+0000 7f545d6971c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:36:56.975 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:56.911+0000 7f545d6971c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:36:57.546 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:57 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4260041679' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-12T08:36:57.546 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:57 smithi090 ceph-mon[29100]: mgrmap e5: y(active, since 6s) 2024-04-12T08:36:57.865 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:57.555+0000 7f545d6971c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:36:57.865 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:57.649+0000 7f545d6971c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:36:57.865 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:57.731+0000 7f545d6971c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:36:58.160 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:57.910+0000 7f545d6971c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:36:58.161 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:57.990+0000 7f545d6971c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:36:58.161 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:58.159+0000 7f545d6971c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:36:58.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-12T08:36:58.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-12T08:36:58.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-12T08:36:58.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-12T08:36:58.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-12T08:36:58.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-12T08:36:58.185 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for the mgr to restart... 2024-04-12T08:36:58.185 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mgr epoch 5... 2024-04-12T08:36:58.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:36:58 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1580956549' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-12T08:36:58.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:58.343+0000 7f545d6971c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:36:58.922 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:58.429+0000 7f545d6971c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:36:58.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:58.522+0000 7f545d6971c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:36:58.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:58.610+0000 7f545d6971c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:36:59.922 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:59 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:59.639+0000 7f545d6971c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:36:59.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:59 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:59.765+0000 7f545d6971c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:37:00.350 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:36:59 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:36:59.955+0000 7f545d6971c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:37:00.350 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:00.177+0000 7f545d6971c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:37:00.350 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:00.349+0000 7f545d6971c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:37:00.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:00.526+0000 7f545d6971c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:37:01.175 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:00.782+0000 7f545d6971c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:37:01.176 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:00.867+0000 7f545d6971c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:37:01.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:01 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:01.252+0000 7f545d6971c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:37:02.304 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:02.017+0000 7f545d6971c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:37:02.304 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:02.303+0000 7f545d6971c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:37:02.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:02.612+0000 7f545d6971c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:37:02.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:02.802+0000 7f545d6971c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:37:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: Active manager daemon y restarted 2024-04-12T08:37:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: Activating manager daemon y 2024-04-12T08:37:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: osdmap e2: 0 total, 0 up, 0 in 2024-04-12T08:37:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: mgrmap e6: y(active, starting, since 0.00309404s) 2024-04-12T08:37:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:37:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:37:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:37:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:37:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:37:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: Manager daemon y is now available 2024-04-12T08:37:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:37:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:02 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:37:04.124 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:03 smithi090 ceph-mon[29100]: Found migration_current of "None". Setting to last migration. 2024-04-12T08:37:04.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:03 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:04.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:03 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:04.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:03 smithi090 ceph-mon[29100]: mgrmap e7: y(active, since 1.00533s) 2024-04-12T08:37:04.387 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-12T08:37:04.387 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-12T08:37:04.387 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-12T08:37:04.387 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-12T08:37:04.387 INFO:teuthology.orchestra.run.smithi090.stdout:mgr epoch 5 is available 2024-04-12T08:37:04.387 INFO:teuthology.orchestra.run.smithi090.stdout:Setting orchestrator backend to cephadm... 2024-04-12T08:37:05.028 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:04 smithi090 ceph-mon[29100]: [12/Apr/2024:08:37:03] ENGINE Bus STARTING 2024-04-12T08:37:05.029 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:04 smithi090 ceph-mon[29100]: [12/Apr/2024:08:37:03] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:37:05.029 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:04 smithi090 ceph-mon[29100]: [12/Apr/2024:08:37:03] ENGINE Bus STARTED 2024-04-12T08:37:05.029 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:04 smithi090 ceph-mon[29100]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-12T08:37:05.029 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:04 smithi090 ceph-mon[29100]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-12T08:37:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:05 smithi090 ceph-mon[29100]: mgrmap e8: y(active, since 2s) 2024-04-12T08:37:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:05 smithi090 ceph-mon[29100]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:05 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:05 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:06.680 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-12T08:37:06.680 INFO:teuthology.orchestra.run.smithi090.stdout:Generating ssh key... 2024-04-12T08:37:07.115 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:06 smithi090 ceph-mon[29100]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: Generating public/private rsa key pair. 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: Your identification has been saved in /tmp/tmp13kivu7e/key. 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: Your public key has been saved in /tmp/tmp13kivu7e/key.pub. 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: The key fingerprint is: 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: SHA256:4VSiVz2E7Z/jr1fA+tz2ckKjejglSPooup7guqCc2JI ceph-96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: The key's randomart image is: 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: +---[RSA 3072]----+ 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: | . o=. | 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: | . +. + | 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: | . + . o | 2024-04-12T08:37:07.791 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: | +.. . o | 2024-04-12T08:37:07.792 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: | oS. o o | 2024-04-12T08:37:07.792 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: | . . . o * .| 2024-04-12T08:37:07.792 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: |o. o + * +.| 2024-04-12T08:37:07.792 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: |Eo... . . o o * =| 2024-04-12T08:37:07.792 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: |*B*o . .+ .O+| 2024-04-12T08:37:07.792 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: +----[SHA256]-----+ 2024-04-12T08:37:08.078 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-mon[29100]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:08.078 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-mon[29100]: Generating ssh key... 2024-04-12T08:37:08.078 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:08.078 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:07 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:09.314 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3qgGf16JfMS+dFG6x5qQxw5Vk6qZVO1Pnd7GgxUl+6RYcCKpXk7szCKtsHUXpU3tDWGbYsiB3ymLxJglzP5SGsbgO/DpTgGLr9I7xU4VbB/TzSs/u1HuTkMoL9LfL9wHfO8mxIQ78667PWnC7hrdLmGqm8L+UlSklnDk6woAKRD+axnsxefdRpZ+6ncInnWOHt0Aw+sfUrQrhh5ZAefB0KRAOwYoOqONSeIaYJHOLiOBfaGlYElGS37FajL//aKt/yrT61yrHp9YZlvODYWnVWuJg+oi51dDkaBbVejBt+B/+blc8cmiTB0wPTTWdm4tlZLvfQ9hh65afyuKp8GUBv2QmaAhd8A7Q0DFsTFNQzmBqtPZwVXMWJaNswEfXOgBoQvwW7o5R+QptILy0V+O2M/DXUuyUQAervIsP2cwQcD/J2T5ALLnltsFPZl8EQMmLqeYMopeGpMA5jjGim17wTcVTslEooiY8GA6/q7bmxDkNzevLASJoa1YFhhn7nj0= ceph-96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:37:09.314 INFO:teuthology.orchestra.run.smithi090.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-12T08:37:09.314 INFO:teuthology.orchestra.run.smithi090.stdout:Adding key to root@localhost authorized_keys... 2024-04-12T08:37:09.315 INFO:teuthology.orchestra.run.smithi090.stdout:Adding host smithi090... 2024-04-12T08:37:09.615 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:09 smithi090 ceph-mon[29100]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:10.558 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:10 smithi090 ceph-mon[29100]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi090", "addr": "172.21.15.90", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:11.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:11 smithi090 ceph-mon[29100]: Deploying cephadm binary to smithi090 2024-04-12T08:37:12.493 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout Added host 'smithi090' with addr '172.21.15.90' 2024-04-12T08:37:12.493 INFO:teuthology.orchestra.run.smithi090.stdout:Deploying unmanaged mon service... 2024-04-12T08:37:13.131 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:12 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:13.131 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:12 smithi090 ceph-mon[29100]: Added host smithi090 2024-04-12T08:37:13.131 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:12 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:13.923 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-12T08:37:13.923 INFO:teuthology.orchestra.run.smithi090.stdout:Deploying unmanaged mgr service... 2024-04-12T08:37:14.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:14 smithi090 ceph-mon[29100]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:14 smithi090 ceph-mon[29100]: Saving service mon spec with placement count:5 2024-04-12T08:37:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:14 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:15.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:15 smithi090 ceph-mon[29100]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:15.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:15 smithi090 ceph-mon[29100]: Saving service mgr spec with placement count:2 2024-04-12T08:37:15.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:15 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:15.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:15 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:16.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-12T08:37:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:16 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:16 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2935750296' entity='client.admin' 2024-04-12T08:37:18.731 INFO:teuthology.orchestra.run.smithi090.stdout:Enabling the dashboard module... 2024-04-12T08:37:19.387 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:19 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1495124789' entity='client.admin' 2024-04-12T08:37:19.387 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:19 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:20.320 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:20 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3122713165' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-12T08:37:20.320 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:20 smithi090 ceph-mon[29100]: from='mgr.14120 172.21.15.90:0/4271563110' entity='mgr.y' 2024-04-12T08:37:20.321 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ignoring --setuser ceph since I am not root 2024-04-12T08:37:20.321 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ignoring --setgroup ceph since I am not root 2024-04-12T08:37:20.630 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:20.540+0000 7f6a7da491c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:37:20.630 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:20.629+0000 7f6a7da491c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:37:21.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:21 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3122713165' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-12T08:37:21.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:21 smithi090 ceph-mon[29100]: mgrmap e9: y(active, since 17s) 2024-04-12T08:37:21.446 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:21.264+0000 7f6a7da491c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:37:21.446 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:21.359+0000 7f6a7da491c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:37:21.446 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:21.444+0000 7f6a7da491c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:37:21.807 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:21.596+0000 7f6a7da491c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:37:21.807 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:21.676+0000 7f6a7da491c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:37:22.116 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:22 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/395289742' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-12T08:37:22.117 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:21.849+0000 7f6a7da491c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:37:22.117 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:22 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:22.031+0000 7f6a7da491c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:37:22.117 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:22 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:22.115+0000 7f6a7da491c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:37:22.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:22 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:22.208+0000 7f6a7da491c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:37:22.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:22 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:22.294+0000 7f6a7da491c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:37:22.902 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-12T08:37:22.902 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-12T08:37:22.902 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-12T08:37:22.902 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-12T08:37:22.902 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-12T08:37:22.902 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-12T08:37:22.902 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for the mgr to restart... 2024-04-12T08:37:22.902 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mgr epoch 9... 2024-04-12T08:37:23.487 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:23.357+0000 7f6a7da491c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:37:23.487 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:23.486+0000 7f6a7da491c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:37:23.906 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:23.678+0000 7f6a7da491c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:37:23.906 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:23.905+0000 7f6a7da491c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:37:24.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:24.081+0000 7f6a7da491c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:37:24.522 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:24.261+0000 7f6a7da491c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:37:24.922 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:24.521+0000 7f6a7da491c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:37:24.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:24.605+0000 7f6a7da491c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:37:25.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:24.992+0000 7f6a7da491c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:37:26.059 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:25.764+0000 7f6a7da491c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:37:26.374 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:26.058+0000 7f6a7da491c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:37:26.374 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:26.373+0000 7f6a7da491c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:37:26.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:37:26.565+0000 7f6a7da491c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:37:26.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: Active manager daemon y restarted 2024-04-12T08:37:26.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: Activating manager daemon y 2024-04-12T08:37:26.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: osdmap e3: 0 total, 0 up, 0 in 2024-04-12T08:37:26.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: mgrmap e10: y(active, starting, since 0.00573721s) 2024-04-12T08:37:26.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:37:26.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:37:26.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:37:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:37:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:37:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: Manager daemon y is now available 2024-04-12T08:37:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:37:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:37:28.171 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:27 smithi090 ceph-mon[29100]: [12/Apr/2024:08:37:26] ENGINE Bus STARTING 2024-04-12T08:37:28.171 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:27 smithi090 ceph-mon[29100]: [12/Apr/2024:08:37:26] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:37:28.171 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:27 smithi090 ceph-mon[29100]: [12/Apr/2024:08:37:26] ENGINE Bus STARTED 2024-04-12T08:37:28.171 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:27 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:28.171 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:27 smithi090 ceph-mon[29100]: mgrmap e11: y(active, since 1.00524s) 2024-04-12T08:37:28.173 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-12T08:37:28.174 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-12T08:37:28.174 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-12T08:37:28.174 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-12T08:37:28.174 INFO:teuthology.orchestra.run.smithi090.stdout:mgr epoch 9 is available 2024-04-12T08:37:28.174 INFO:teuthology.orchestra.run.smithi090.stdout:Generating a dashboard self-signed certificate... 2024-04-12T08:37:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:28 smithi090 ceph-mon[29100]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-12T08:37:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:28 smithi090 ceph-mon[29100]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-12T08:37:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:29.434 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-12T08:37:29.434 INFO:teuthology.orchestra.run.smithi090.stdout:Creating initial admin user... 2024-04-12T08:37:30.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:29 smithi090 ceph-mon[29100]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:30.126 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:29 smithi090 ceph-mon[29100]: mgrmap e12: y(active, since 2s) 2024-04-12T08:37:30.126 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:29 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:31.107 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$0Se0LTOiR26MPmwRJKwyCenBI6eQxG/lpdXqA1FUeu1x1RnbdUwSe", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712911050, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-12T08:37:31.107 INFO:teuthology.orchestra.run.smithi090.stdout:Fetching dashboard port number... 2024-04-12T08:37:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:31 smithi090 ceph-mon[29100]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:32.557 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:32 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3373071783' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-12T08:37:32.560 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 8443 2024-04-12T08:37:32.570 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-12T08:37:32.571 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout disabled 2024-04-12T08:37:32.579 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-12T08:37:32.579 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-12T08:37:32.579 INFO:teuthology.orchestra.run.smithi090.stdout:firewalld.service is not enabled 2024-04-12T08:37:32.579 INFO:teuthology.orchestra.run.smithi090.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-12T08:37:32.581 INFO:teuthology.orchestra.run.smithi090.stdout:Ceph Dashboard is now available at: 2024-04-12T08:37:32.581 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:32.581 INFO:teuthology.orchestra.run.smithi090.stdout: URL: https://smithi090.front.sepia.ceph.com:8443/ 2024-04-12T08:37:32.581 INFO:teuthology.orchestra.run.smithi090.stdout: User: admin 2024-04-12T08:37:32.582 INFO:teuthology.orchestra.run.smithi090.stdout: Password: xjei92yzeg 2024-04-12T08:37:32.582 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:32.582 INFO:teuthology.orchestra.run.smithi090.stdout:Saving cluster configuration to /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config directory 2024-04-12T08:37:32.583 INFO:teuthology.orchestra.run.smithi090.stdout:Enabling autotune for osd_memory_target 2024-04-12T08:37:34.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:34 smithi090 ceph-mon[29100]: mgrmap e13: y(active, since 6s) 2024-04-12T08:37:35.070 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-12T08:37:35.070 INFO:teuthology.orchestra.run.smithi090.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-12T08:37:35.070 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout:Or, if you are only running a single cluster on this host: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: ceph telemetry on 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout:For more information see: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:35.071 INFO:teuthology.orchestra.run.smithi090.stdout:Bootstrap complete. 2024-04-12T08:37:35.108 INFO:tasks.cephadm:Fetching config... 2024-04-12T08:37:35.108 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:37:35.108 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-12T08:37:35.123 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-12T08:37:35.123 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:37:35.123 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-12T08:37:35.179 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-12T08:37:35.179 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:37:35.179 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd if=/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-12T08:37:35.246 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-12T08:37:35.246 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:37:35.246 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-12T08:37:35.302 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-12T08:37:35.302 DEBUG:teuthology.orchestra.run.smithi090:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3qgGf16JfMS+dFG6x5qQxw5Vk6qZVO1Pnd7GgxUl+6RYcCKpXk7szCKtsHUXpU3tDWGbYsiB3ymLxJglzP5SGsbgO/DpTgGLr9I7xU4VbB/TzSs/u1HuTkMoL9LfL9wHfO8mxIQ78667PWnC7hrdLmGqm8L+UlSklnDk6woAKRD+axnsxefdRpZ+6ncInnWOHt0Aw+sfUrQrhh5ZAefB0KRAOwYoOqONSeIaYJHOLiOBfaGlYElGS37FajL//aKt/yrT61yrHp9YZlvODYWnVWuJg+oi51dDkaBbVejBt+B/+blc8cmiTB0wPTTWdm4tlZLvfQ9hh65afyuKp8GUBv2QmaAhd8A7Q0DFsTFNQzmBqtPZwVXMWJaNswEfXOgBoQvwW7o5R+QptILy0V+O2M/DXUuyUQAervIsP2cwQcD/J2T5ALLnltsFPZl8EQMmLqeYMopeGpMA5jjGim17wTcVTslEooiY8GA6/q7bmxDkNzevLASJoa1YFhhn7nj0= ceph-96735376-f8a7-11ee-bc8c-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-12T08:37:35.388 INFO:teuthology.orchestra.run.smithi090.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3qgGf16JfMS+dFG6x5qQxw5Vk6qZVO1Pnd7GgxUl+6RYcCKpXk7szCKtsHUXpU3tDWGbYsiB3ymLxJglzP5SGsbgO/DpTgGLr9I7xU4VbB/TzSs/u1HuTkMoL9LfL9wHfO8mxIQ78667PWnC7hrdLmGqm8L+UlSklnDk6woAKRD+axnsxefdRpZ+6ncInnWOHt0Aw+sfUrQrhh5ZAefB0KRAOwYoOqONSeIaYJHOLiOBfaGlYElGS37FajL//aKt/yrT61yrHp9YZlvODYWnVWuJg+oi51dDkaBbVejBt+B/+blc8cmiTB0wPTTWdm4tlZLvfQ9hh65afyuKp8GUBv2QmaAhd8A7Q0DFsTFNQzmBqtPZwVXMWJaNswEfXOgBoQvwW7o5R+QptILy0V+O2M/DXUuyUQAervIsP2cwQcD/J2T5ALLnltsFPZl8EQMmLqeYMopeGpMA5jjGim17wTcVTslEooiY8GA6/q7bmxDkNzevLASJoa1YFhhn7nj0= ceph-96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:37:35.404 DEBUG:teuthology.orchestra.run.smithi114:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3qgGf16JfMS+dFG6x5qQxw5Vk6qZVO1Pnd7GgxUl+6RYcCKpXk7szCKtsHUXpU3tDWGbYsiB3ymLxJglzP5SGsbgO/DpTgGLr9I7xU4VbB/TzSs/u1HuTkMoL9LfL9wHfO8mxIQ78667PWnC7hrdLmGqm8L+UlSklnDk6woAKRD+axnsxefdRpZ+6ncInnWOHt0Aw+sfUrQrhh5ZAefB0KRAOwYoOqONSeIaYJHOLiOBfaGlYElGS37FajL//aKt/yrT61yrHp9YZlvODYWnVWuJg+oi51dDkaBbVejBt+B/+blc8cmiTB0wPTTWdm4tlZLvfQ9hh65afyuKp8GUBv2QmaAhd8A7Q0DFsTFNQzmBqtPZwVXMWJaNswEfXOgBoQvwW7o5R+QptILy0V+O2M/DXUuyUQAervIsP2cwQcD/J2T5ALLnltsFPZl8EQMmLqeYMopeGpMA5jjGim17wTcVTslEooiY8GA6/q7bmxDkNzevLASJoa1YFhhn7nj0= ceph-96735376-f8a7-11ee-bc8c-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-12T08:37:35.449 INFO:teuthology.orchestra.run.smithi114.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3qgGf16JfMS+dFG6x5qQxw5Vk6qZVO1Pnd7GgxUl+6RYcCKpXk7szCKtsHUXpU3tDWGbYsiB3ymLxJglzP5SGsbgO/DpTgGLr9I7xU4VbB/TzSs/u1HuTkMoL9LfL9wHfO8mxIQ78667PWnC7hrdLmGqm8L+UlSklnDk6woAKRD+axnsxefdRpZ+6ncInnWOHt0Aw+sfUrQrhh5ZAefB0KRAOwYoOqONSeIaYJHOLiOBfaGlYElGS37FajL//aKt/yrT61yrHp9YZlvODYWnVWuJg+oi51dDkaBbVejBt+B/+blc8cmiTB0wPTTWdm4tlZLvfQ9hh65afyuKp8GUBv2QmaAhd8A7Q0DFsTFNQzmBqtPZwVXMWJaNswEfXOgBoQvwW7o5R+QptILy0V+O2M/DXUuyUQAervIsP2cwQcD/J2T5ALLnltsFPZl8EQMmLqeYMopeGpMA5jjGim17wTcVTslEooiY8GA6/q7bmxDkNzevLASJoa1YFhhn7nj0= ceph-96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:37:35.465 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-12T08:37:35.648 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:35 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2265987971' entity='client.admin' 2024-04-12T08:37:35.718 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:37:38.280 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:38 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:38.280 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:38 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:38.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:38 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi090", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:37:38.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:38 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:38.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:38 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:38.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:38 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:39.236 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-12T08:37:39.236 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-12T08:37:39.719 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:37:39.834 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:39 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3284067647' entity='client.admin' 2024-04-12T08:37:39.834 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:39 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:39.834 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:39 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:42.213 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi114 2024-04-12T08:37:42.213 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:37:42.213 DEBUG:teuthology.orchestra.run.smithi114:> dd of=/etc/ceph/ceph.conf 2024-04-12T08:37:42.234 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:37:42.234 DEBUG:teuthology.orchestra.run.smithi114:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:37:42.291 INFO:tasks.cephadm:Adding host smithi114 to orchestrator... 2024-04-12T08:37:42.292 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch host add smithi114 2024-04-12T08:37:42.542 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:37:42.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:42.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:42.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:42.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:37:42.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:37:42.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:37:42.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:37:42.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:37:42.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:42.769 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:42.769 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:42 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:43.580 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:43 smithi090 ceph-mon[29100]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:37:45.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:45 smithi090 ceph-mon[29100]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi114", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:45.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:45 smithi090 ceph-mon[29100]: Deploying cephadm binary to smithi114 2024-04-12T08:37:46.269 INFO:teuthology.orchestra.run.smithi090.stdout:Added host 'smithi114' with addr '172.21.15.114' 2024-04-12T08:37:46.865 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch host ls --format=json 2024-04-12T08:37:47.114 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:37:47.493 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:47 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:47.493 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:47 smithi090 ceph-mon[29100]: Added host smithi114 2024-04-12T08:37:47.493 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:47 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:47.493 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:47 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:47.493 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:47 smithi090 ceph-mon[29100]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:37:47.493 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:47 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:48.857 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:37:48.858 INFO:teuthology.orchestra.run.smithi090.stdout:[{"addr": "172.21.15.90", "hostname": "smithi090", "labels": [], "status": ""}, {"addr": "172.21.15.114", "hostname": "smithi114", "labels": [], "status": ""}] 2024-04-12T08:37:48.858 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:48 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:49.501 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-12T08:37:49.501 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd crush tunables default 2024-04-12T08:37:49.752 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:37:49.874 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:49 smithi090 ceph-mon[29100]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:37:49.874 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:49 smithi090 ceph-mon[29100]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:37:51.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:51.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:51.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:51.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:51.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:37:51.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:37:51.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:37:51.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:37:51.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:37:51.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:37:51.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:37:51.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:51.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1616079656' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-12T08:37:51.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:51.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:52.489 INFO:teuthology.orchestra.run.smithi090.stderr:adjusted tunables profile to default 2024-04-12T08:37:52.802 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:52 smithi090 ceph-mon[29100]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:37:52.802 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:52 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1616079656' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-12T08:37:52.802 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:52 smithi090 ceph-mon[29100]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T08:37:53.091 INFO:tasks.cephadm:Adding mon.a on smithi090 2024-04-12T08:37:53.091 INFO:tasks.cephadm:Adding mon.c on smithi090 2024-04-12T08:37:53.091 INFO:tasks.cephadm:Adding mon.b on smithi114 2024-04-12T08:37:53.092 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch apply mon '3;smithi090:172.21.15.90=a;smithi090:[v2:172.21.15.90:3301,v1:172.21.15.90:6790]=c;smithi114:172.21.15.114=b' 2024-04-12T08:37:53.341 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:37:53.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:53 smithi090 ceph-mon[29100]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:37:55.072 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled mon update... 2024-04-12T08:37:55.661 DEBUG:teuthology.orchestra.run.smithi090:mon.c> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.c.service 2024-04-12T08:37:55.665 DEBUG:teuthology.orchestra.run.smithi114:mon.b> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.b.service 2024-04-12T08:37:55.667 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-12T08:37:55.667 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph mon dump -f json 2024-04-12T08:37:56.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:56 smithi090 ceph-mon[29100]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:37:56.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:56 smithi090 ceph-mon[29100]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi090:172.21.15.90=a;smithi090:[v2:172.21.15.90:3301,v1:172.21.15.90:6790]=c;smithi114:172.21.15.114=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:37:56.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:56 smithi090 ceph-mon[29100]: Saving service mon spec with placement smithi090:172.21.15.90=a;smithi090:[v2:172.21.15.90:3301,v1:172.21.15.90:6790]=c;smithi114:172.21.15.114=b;count:3 2024-04-12T08:37:56.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:56.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:37:56.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:37:56.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:37:56.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:37:56.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:37:56.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:37:57.233 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:37:57.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:57 smithi090 ceph-mon[29100]: Deploying daemon mon.b on smithi114 2024-04-12T08:37:58.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:58 smithi090 ceph-mon[29100]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:37:59.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:37:59 smithi090 ceph-mon[29100]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:37:59.579 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:37:59 smithi114 systemd[1]: Started Ceph mon.b for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:38:04.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: mon.a calling monitor election 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: mon.b calling monitor election 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: monmap e2: 2 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: fsmap 2024-04-12T08:38:04.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T08:38:04.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: mgrmap e13: y(active, since 37s) 2024-04-12T08:38:04.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: overall HEALTH_OK 2024-04-12T08:38:04.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:04.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: mon.a calling monitor election 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: mon.b calling monitor election 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-12T08:38:04.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: monmap e2: 2 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T08:38:04.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: fsmap 2024-04-12T08:38:04.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T08:38:04.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: mgrmap e13: y(active, since 37s) 2024-04-12T08:38:04.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: overall HEALTH_OK 2024-04-12T08:38:04.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:04.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:05.782 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:05 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:05.782 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:05 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:05.782 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:05 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:05.782 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:05 smithi114 ceph-mon[33573]: Deploying daemon mon.c on smithi090 2024-04-12T08:38:05.782 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:05 smithi114 ceph-mon[33573]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:05.782 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:05 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:05.854 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:05 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:05.854 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:05 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:05.854 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:05 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:05.854 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:05 smithi090 ceph-mon[29100]: Deploying daemon mon.c on smithi090 2024-04-12T08:38:05.854 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:05 smithi090 ceph-mon[29100]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:05.854 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:05 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:05.964 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:38:05.965 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":2,"fsid":"96735376-f8a7-11ee-bc8c-c7b262605968","modified":"2024-04-12T08:37:59.458317Z","created":"2024-04-12T08:36:31.874980Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:3300","nonce":0},{"type":"v1","addr":"172.21.15.90:6789","nonce":0}]},"addr":"172.21.15.90:6789/0","public_addr":"172.21.15.90:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:3300","nonce":0},{"type":"v1","addr":"172.21.15.114:6789","nonce":0}]},"addr":"172.21.15.114:6789/0","public_addr":"172.21.15.114:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-12T08:38:05.966 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 2 2024-04-12T08:38:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:06 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/3474610545' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T08:38:06.921 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:06 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/3474610545' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T08:38:07.506 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-12T08:38:07.506 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph mon dump -f json 2024-04-12T08:38:07.561 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[29100]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:07.562 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 podman[39974]: 2024-04-12 08:38:07.548512685 +0000 UTC m=+0.136182890 container create d5aa934587fe7593704ae36a833843ee78f09f16def743564558edcb3e58d193 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-12T08:38:07.758 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:38:07.816 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 podman[39974]: 2024-04-12 08:38:07.793281563 +0000 UTC m=+0.380951771 container init d5aa934587fe7593704ae36a833843ee78f09f16def743564558edcb3e58d193 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:38:07.817 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 podman[39974]: 2024-04-12 08:38:07.796356602 +0000 UTC m=+0.384026808 container start d5aa934587fe7593704ae36a833843ee78f09f16def743564558edcb3e58d193 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-d3b5060, ceph=True) 2024-04-12T08:38:07.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:07 smithi114 ceph-mon[33573]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T08:38:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable), process ceph-mon, pid 2 2024-04-12T08:38:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: pidfile_write: ignore empty --pid-file 2024-04-12T08:38:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: load: jerasure load: lrc 2024-04-12T08:38:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: RocksDB version: 6.15.5 2024-04-12T08:38:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-12T08:38:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Compile date Apr 9 2024 2024-04-12T08:38:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: DB SUMMARY 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: DB Session ID: 9MZ8VKKSD3V5NKIN73IF 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: CURRENT file: CURRENT 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.error_if_exists: 0 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.create_if_missing: 0 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.paranoid_checks: 1 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.env: 0x5631c066aa80 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.fs: Posix File System 2024-04-12T08:38:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.info_log: 0x5631c1570e80 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.statistics: (nil) 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.use_fsync: 0 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_log_file_size: 0 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.allow_fallocate: 1 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.use_direct_reads: 0 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.db_log_dir: 2024-04-12T08:38:08.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-12T08:38:08.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T08:38:08.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T08:38:08.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T08:38:08.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.write_buffer_manager: 0x5631c22e0930 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T08:38:08.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.unordered_write: 0 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.row_cache: None 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.wal_filter: None 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.preserve_deletes: 0 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.two_write_queues: 0 2024-04-12T08:38:08.179 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.atomic_flush: 0 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.log_readahead_size: 0 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_background_jobs: 2 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_background_compactions: -1 2024-04-12T08:38:08.180 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_subcompactions: 1 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_open_files: -1 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_background_flushes: -1 2024-04-12T08:38:08.181 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Compression algorithms supported: 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: kZSTD supported: 0 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: kXpressCompression supported: 0 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: kZlibCompression supported: 1 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: kSnappyCompression supported: 1 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: kLZ4Compression supported: 1 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: kBZip2Compression supported: 0 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.merge_operator: 2024-04-12T08:38:08.182 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_filter: None 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5631c15c0f58) 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout: cache_index_and_filter_blocks: 1 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout: pin_top_level_index_and_filter: 1 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout: index_type: 0 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout: data_block_index_type: 0 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout: index_shortening: 1 2024-04-12T08:38:08.183 INFO:journalctl@ceph.mon.c.smithi090.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: hash_index_allow_collision: 1 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: checksum: 1 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: no_block_cache: 0 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_cache: 0x5631c15f3090 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_cache_name: BinnedLRUCache 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_cache_options: 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: capacity : 536870912 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: num_shard_bits : 4 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: strict_capacity_limit : 0 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: high_pri_pool_ratio: 0.000 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_cache_compressed: (nil) 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: persistent_cache: (nil) 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_size: 4096 2024-04-12T08:38:08.184 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_size_deviation: 10 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_restart_interval: 16 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: index_block_restart_interval: 1 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: metadata_block_size: 4096 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: partition_filters: 0 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: use_delta_encoding: 1 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: whole_key_filtering: 1 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: verify_compression: 0 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: read_amp_bytes_per_bit: 0 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: format_version: 4 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: enable_index_compression: 1 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_align: 0 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T08:38:08.185 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compression: NoCompression 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.num_levels: 7 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T08:38:08.186 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T08:38:08.187 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.arena_block_size: 4194304 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-12T08:38:08.188 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.table_properties_collectors: 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.inplace_update_support: 0 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T08:38:08.189 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.bloom_locality: 0 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.max_successive_merges: 0 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.ttl: 2592000 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.enable_blob_files: false 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.min_blob_size: 0 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T08:38:08.190 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911087823011, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911087823311, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712911087, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "747a31ab-5eb5-4e39-9784-9ca0f00de54d", "db_session_id": "9MZ8VKKSD3V5NKIN73IF"}} 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911087823622, "job": 1, "event": "recovery_finished"} 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5631c1587180 2024-04-12T08:38:08.191 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: DB pointer 0x5631c165c000 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** DB Stats ** 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:38:08.192 INFO:journalctl@ceph.mon.c.smithi090.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T08:38:08.193 INFO:journalctl@ceph.mon.c.smithi090.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: Interval compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:38:08.194 INFO:journalctl@ceph.mon.c.smithi090.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-12T08:38:08.195 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: using public_addrv [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: starting mon.c rank -1 at public addrs [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] at bind addrs [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(???) e0 preinit fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).mds e1 new map 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout: e1 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout: legacy client fscid: -1 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout: No filesystems configured 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-12T08:38:08.196 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:38:08.197 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1616079656' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1616079656' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi090:172.21.15.90=a;smithi090:[v2:172.21.15.90:3301,v1:172.21.15.90:6790]=c;smithi114:172.21.15.114=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: Saving service mon spec with placement smithi090:172.21.15.90=a;smithi090:[v2:172.21.15.90:3301,v1:172.21.15.90:6790]=c;smithi114:172.21.15.114=b;count:3 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:08.198 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: Deploying daemon mon.b on smithi114 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.a calling monitor election 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.b calling monitor election 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:08.199 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:08.202 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:08.202 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:08.202 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-12T08:38:08.202 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: monmap e2: 2 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T08:38:08.202 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: fsmap 2024-04-12T08:38:08.202 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T08:38:08.202 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mgrmap e13: y(active, since 37s) 2024-04-12T08:38:08.202 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: overall HEALTH_OK 2024-04-12T08:38:08.202 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: Deploying daemon mon.c on smithi090 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/3474610545' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 ceph-mon[40003]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 bash[39974]: d5aa934587fe7593704ae36a833843ee78f09f16def743564558edcb3e58d193 2024-04-12T08:38:08.203 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:07 smithi090 systemd[1]: Started Ceph mon.c for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:38:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:38:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: mon.a calling monitor election 2024-04-12T08:38:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: mon.b calling monitor election 2024-04-12T08:38:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: mon.c calling monitor election 2024-04-12T08:38:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: monmap e3: 3 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],c=[v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: fsmap 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: mgrmap e13: y(active, since 46s) 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: overall HEALTH_OK 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: mon.a calling monitor election 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: mon.b calling monitor election 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: mon.c calling monitor election 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:38:13.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: monmap e3: 3 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],c=[v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T08:38:13.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: fsmap 2024-04-12T08:38:13.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T08:38:13.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: mgrmap e13: y(active, since 46s) 2024-04-12T08:38:13.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: overall HEALTH_OK 2024-04-12T08:38:13.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:12 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:38:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: mon.a calling monitor election 2024-04-12T08:38:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:38:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: mon.b calling monitor election 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: mon.c calling monitor election 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: monmap e3: 3 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],c=[v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: fsmap 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: mgrmap e13: y(active, since 46s) 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: overall HEALTH_OK 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:13.789 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-12T08:38:13.790 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":3,"fsid":"96735376-f8a7-11ee-bc8c-c7b262605968","modified":"2024-04-12T08:38:07.849098Z","created":"2024-04-12T08:36:31.874980Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:3300","nonce":0},{"type":"v1","addr":"172.21.15.90:6789","nonce":0}]},"addr":"172.21.15.90:6789/0","public_addr":"172.21.15.90:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:3300","nonce":0},{"type":"v1","addr":"172.21.15.114:6789","nonce":0}]},"addr":"172.21.15.114:6789/0","public_addr":"172.21.15.114:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:3301","nonce":0},{"type":"v1","addr":"172.21.15.90:6790","nonce":0}]},"addr":"172.21.15.90:6790/0","public_addr":"172.21.15.90:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-12T08:38:13.792 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 3 2024-04-12T08:38:14.358 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:13 smithi114 ceph-mon[33573]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:14.358 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:13 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:14.358 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:13 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/335429803' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T08:38:14.358 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:13 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:14.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:13 smithi090 ceph-mon[40003]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:14.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:13 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:14.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:13 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/335429803' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T08:38:14.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:13 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:14.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:13 smithi090 ceph-mon[29100]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:14.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:13 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:14.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:13 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/335429803' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T08:38:14.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:13 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:38:14.684 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-12T08:38:14.685 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph config generate-minimal-conf 2024-04-12T08:38:14.942 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:38:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:38:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:15 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:15.372 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:38:15.373 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:38:15.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:15 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:16.358 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[29100]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:38:16.358 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[29100]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:38:16.358 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[29100]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:38:16.358 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[29100]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:38:16.358 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[29100]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:16.358 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[29100]: Reconfiguring mon.a (unknown last config time)... 2024-04-12T08:38:16.358 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[29100]: Reconfiguring daemon mon.a on smithi090 2024-04-12T08:38:16.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[40003]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:38:16.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:38:16.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:38:16.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[40003]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:38:16.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[40003]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:16.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[40003]: Reconfiguring mon.a (unknown last config time)... 2024-04-12T08:38:16.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:16 smithi090 ceph-mon[40003]: Reconfiguring daemon mon.a on smithi090 2024-04-12T08:38:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:16 smithi114 ceph-mon[33573]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:38:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:16 smithi114 ceph-mon[33573]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:38:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:16 smithi114 ceph-mon[33573]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:38:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:16 smithi114 ceph-mon[33573]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:38:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:16 smithi114 ceph-mon[33573]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:16 smithi114 ceph-mon[33573]: Reconfiguring mon.a (unknown last config time)... 2024-04-12T08:38:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:16 smithi114 ceph-mon[33573]: Reconfiguring daemon mon.a on smithi090 2024-04-12T08:38:17.720 INFO:teuthology.orchestra.run.smithi090.stdout:# minimal ceph.conf for 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:38:17.720 INFO:teuthology.orchestra.run.smithi090.stdout:[global] 2024-04-12T08:38:17.720 INFO:teuthology.orchestra.run.smithi090.stdout: fsid = 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:38:17.720 INFO:teuthology.orchestra.run.smithi090.stdout: mon_host = [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] 2024-04-12T08:38:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:17 smithi114 ceph-mon[33573]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:17 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:17 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:17 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:17 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:17 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:38:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:17 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[29100]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:38:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:17.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[40003]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:38:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:17 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:18.297 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-12T08:38:18.297 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:38:18.298 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/etc/ceph/ceph.conf 2024-04-12T08:38:18.327 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:38:18.327 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:38:18.394 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:38:18.394 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/ceph/ceph.conf 2024-04-12T08:38:18.429 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:38:18.429 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:38:18.499 INFO:tasks.cephadm:Adding mgr.y on smithi090 2024-04-12T08:38:18.499 INFO:tasks.cephadm:Adding mgr.x on smithi114 2024-04-12T08:38:18.499 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch apply mgr '2;smithi090=y;smithi114=x' 2024-04-12T08:38:18.783 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:38:18.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:18 smithi114 ceph-mon[33573]: Reconfiguring mon.c (monmap changed)... 2024-04-12T08:38:18.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:18 smithi114 ceph-mon[33573]: Reconfiguring daemon mon.c on smithi090 2024-04-12T08:38:18.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:18 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/625873495' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:18.878 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:18 smithi090 ceph-mon[29100]: Reconfiguring mon.c (monmap changed)... 2024-04-12T08:38:18.879 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:18 smithi090 ceph-mon[29100]: Reconfiguring daemon mon.c on smithi090 2024-04-12T08:38:18.879 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:18 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/625873495' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:18.879 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:18 smithi090 ceph-mon[40003]: Reconfiguring mon.c (monmap changed)... 2024-04-12T08:38:18.879 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:18 smithi090 ceph-mon[40003]: Reconfiguring daemon mon.c on smithi090 2024-04-12T08:38:18.879 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:18 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/625873495' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:19.911 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:19 smithi114 ceph-mon[33573]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:19.911 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:19 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:19.911 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:19 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:19.911 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:19 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:19.911 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:19 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:38:19.911 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:19 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:19.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[29100]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[40003]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:38:19.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:19 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:20.512 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled mgr update... 2024-04-12T08:38:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:20 smithi114 ceph-mon[33573]: Reconfiguring mon.b (monmap changed)... 2024-04-12T08:38:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:20 smithi114 ceph-mon[33573]: Reconfiguring daemon mon.b on smithi114 2024-04-12T08:38:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:20 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:20 smithi090 ceph-mon[29100]: Reconfiguring mon.b (monmap changed)... 2024-04-12T08:38:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:20 smithi090 ceph-mon[29100]: Reconfiguring daemon mon.b on smithi114 2024-04-12T08:38:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:20 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:20 smithi090 ceph-mon[40003]: Reconfiguring mon.b (monmap changed)... 2024-04-12T08:38:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:20 smithi090 ceph-mon[40003]: Reconfiguring daemon mon.b on smithi114 2024-04-12T08:38:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:20 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:21.161 DEBUG:teuthology.orchestra.run.smithi114:mgr.x> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.x.service 2024-04-12T08:38:21.163 INFO:tasks.cephadm:Deploying OSDs... 2024-04-12T08:38:21.164 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:38:21.164 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/scratch_devs of=/dev/stdout 2024-04-12T08:38:21.183 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-12T08:38:21.183 DEBUG:teuthology.orchestra.run.smithi090:> stat /dev/vg_nvme/lv_1 2024-04-12T08:38:21.239 INFO:teuthology.orchestra.run.smithi090.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-12T08:38:21.240 INFO:teuthology.orchestra.run.smithi090.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T08:38:21.240 INFO:teuthology.orchestra.run.smithi090.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-12T08:38:21.240 INFO:teuthology.orchestra.run.smithi090.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T08:38:21.240 INFO:teuthology.orchestra.run.smithi090.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T08:38:21.240 INFO:teuthology.orchestra.run.smithi090.stdout:Access: 2024-04-12 08:38:07.241089042 +0000 2024-04-12T08:38:21.240 INFO:teuthology.orchestra.run.smithi090.stdout:Modify: 2024-04-12 08:32:06.963803339 +0000 2024-04-12T08:38:21.240 INFO:teuthology.orchestra.run.smithi090.stdout:Change: 2024-04-12 08:32:06.963803339 +0000 2024-04-12T08:38:21.240 INFO:teuthology.orchestra.run.smithi090.stdout: Birth: 2024-04-12 08:32:06.963803339 +0000 2024-04-12T08:38:21.240 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-12T08:38:21.309 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records in 2024-04-12T08:38:21.309 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records out 2024-04-12T08:38:21.309 INFO:teuthology.orchestra.run.smithi090.stderr:512 bytes copied, 0.000280331 s, 1.8 MB/s 2024-04-12T08:38:21.310 DEBUG:teuthology.orchestra.run.smithi090:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-12T08:38:21.368 DEBUG:teuthology.orchestra.run.smithi090:> stat /dev/vg_nvme/lv_2 2024-04-12T08:38:21.424 INFO:teuthology.orchestra.run.smithi090.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-12T08:38:21.425 INFO:teuthology.orchestra.run.smithi090.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T08:38:21.425 INFO:teuthology.orchestra.run.smithi090.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-12T08:38:21.425 INFO:teuthology.orchestra.run.smithi090.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T08:38:21.425 INFO:teuthology.orchestra.run.smithi090.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T08:38:21.425 INFO:teuthology.orchestra.run.smithi090.stdout:Access: 2024-04-12 08:38:07.241089042 +0000 2024-04-12T08:38:21.425 INFO:teuthology.orchestra.run.smithi090.stdout:Modify: 2024-04-12 08:32:07.291798096 +0000 2024-04-12T08:38:21.425 INFO:teuthology.orchestra.run.smithi090.stdout:Change: 2024-04-12 08:32:07.291798096 +0000 2024-04-12T08:38:21.425 INFO:teuthology.orchestra.run.smithi090.stdout: Birth: 2024-04-12 08:32:07.291798096 +0000 2024-04-12T08:38:21.425 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-12T08:38:21.492 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records in 2024-04-12T08:38:21.492 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records out 2024-04-12T08:38:21.492 INFO:teuthology.orchestra.run.smithi090.stderr:512 bytes copied, 0.000203736 s, 2.5 MB/s 2024-04-12T08:38:21.493 DEBUG:teuthology.orchestra.run.smithi090:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-12T08:38:21.550 DEBUG:teuthology.orchestra.run.smithi090:> stat /dev/vg_nvme/lv_3 2024-04-12T08:38:21.606 INFO:teuthology.orchestra.run.smithi090.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-12T08:38:21.606 INFO:teuthology.orchestra.run.smithi090.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T08:38:21.606 INFO:teuthology.orchestra.run.smithi090.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-12T08:38:21.607 INFO:teuthology.orchestra.run.smithi090.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T08:38:21.607 INFO:teuthology.orchestra.run.smithi090.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T08:38:21.607 INFO:teuthology.orchestra.run.smithi090.stdout:Access: 2024-04-12 08:38:07.242089026 +0000 2024-04-12T08:38:21.607 INFO:teuthology.orchestra.run.smithi090.stdout:Modify: 2024-04-12 08:32:07.620792836 +0000 2024-04-12T08:38:21.607 INFO:teuthology.orchestra.run.smithi090.stdout:Change: 2024-04-12 08:32:07.620792836 +0000 2024-04-12T08:38:21.607 INFO:teuthology.orchestra.run.smithi090.stdout: Birth: 2024-04-12 08:32:07.620792836 +0000 2024-04-12T08:38:21.607 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-12T08:38:21.673 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records in 2024-04-12T08:38:21.673 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records out 2024-04-12T08:38:21.673 INFO:teuthology.orchestra.run.smithi090.stderr:512 bytes copied, 0.000231995 s, 2.2 MB/s 2024-04-12T08:38:21.675 DEBUG:teuthology.orchestra.run.smithi090:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-12T08:38:21.732 DEBUG:teuthology.orchestra.run.smithi090:> stat /dev/vg_nvme/lv_4 2024-04-12T08:38:21.787 INFO:teuthology.orchestra.run.smithi090.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-12T08:38:21.787 INFO:teuthology.orchestra.run.smithi090.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T08:38:21.787 INFO:teuthology.orchestra.run.smithi090.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-04-12T08:38:21.787 INFO:teuthology.orchestra.run.smithi090.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T08:38:21.787 INFO:teuthology.orchestra.run.smithi090.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T08:38:21.787 INFO:teuthology.orchestra.run.smithi090.stdout:Access: 2024-04-12 08:38:07.242089026 +0000 2024-04-12T08:38:21.787 INFO:teuthology.orchestra.run.smithi090.stdout:Modify: 2024-04-12 08:32:07.951787544 +0000 2024-04-12T08:38:21.787 INFO:teuthology.orchestra.run.smithi090.stdout:Change: 2024-04-12 08:32:07.951787544 +0000 2024-04-12T08:38:21.787 INFO:teuthology.orchestra.run.smithi090.stdout: Birth: 2024-04-12 08:32:07.951787544 +0000 2024-04-12T08:38:21.787 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-12T08:38:21.852 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records in 2024-04-12T08:38:21.852 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records out 2024-04-12T08:38:21.852 INFO:teuthology.orchestra.run.smithi090.stderr:512 bytes copied, 0.000223644 s, 2.3 MB/s 2024-04-12T08:38:21.853 DEBUG:teuthology.orchestra.run.smithi090:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-12T08:38:21.909 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:38:21.910 DEBUG:teuthology.orchestra.run.smithi114:> dd if=/scratch_devs of=/dev/stdout 2024-04-12T08:38:21.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:21 smithi090 ceph-mon[29100]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi090=y;smithi114=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:38:21.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:21 smithi090 ceph-mon[29100]: Saving service mgr spec with placement smithi090=y;smithi114=x;count:2 2024-04-12T08:38:21.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:21 smithi090 ceph-mon[29100]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:21 smithi090 ceph-mon[40003]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi090=y;smithi114=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:38:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:21 smithi090 ceph-mon[40003]: Saving service mgr spec with placement smithi090=y;smithi114=x;count:2 2024-04-12T08:38:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:21 smithi090 ceph-mon[40003]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:21.928 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-12T08:38:21.929 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_1 2024-04-12T08:38:21.964 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:21 smithi114 ceph-mon[33573]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi090=y;smithi114=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:38:21.965 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:21 smithi114 ceph-mon[33573]: Saving service mgr spec with placement smithi090=y;smithi114=x;count:2 2024-04-12T08:38:21.965 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:21 smithi114 ceph-mon[33573]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:21.983 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-12T08:38:21.983 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T08:38:21.983 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-04-12T08:38:21.983 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T08:38:21.983 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T08:38:21.983 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-04-12 08:37:58.681906163 +0000 2024-04-12T08:38:21.983 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-04-12 08:31:59.086066787 +0000 2024-04-12T08:38:21.983 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-04-12 08:31:59.086066787 +0000 2024-04-12T08:38:21.983 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-04-12 08:31:59.086066787 +0000 2024-04-12T08:38:21.984 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-12T08:38:22.051 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-04-12T08:38:22.052 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-04-12T08:38:22.052 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000855634 s, 598 kB/s 2024-04-12T08:38:22.053 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-12T08:38:22.108 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_2 2024-04-12T08:38:22.163 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-12T08:38:22.163 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T08:38:22.163 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-04-12T08:38:22.163 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T08:38:22.163 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T08:38:22.163 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-04-12 08:37:58.682906141 +0000 2024-04-12T08:38:22.163 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-04-12 08:31:59.428059290 +0000 2024-04-12T08:38:22.163 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-04-12 08:31:59.428059290 +0000 2024-04-12T08:38:22.163 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-04-12 08:31:59.428059290 +0000 2024-04-12T08:38:22.164 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-12T08:38:22.229 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-04-12T08:38:22.229 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-04-12T08:38:22.229 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000225371 s, 2.3 MB/s 2024-04-12T08:38:22.230 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-12T08:38:22.286 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_3 2024-04-12T08:38:22.343 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-12T08:38:22.343 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T08:38:22.343 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-12T08:38:22.343 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T08:38:22.343 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T08:38:22.343 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-04-12 08:37:58.682906141 +0000 2024-04-12T08:38:22.343 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-04-12 08:31:59.769051815 +0000 2024-04-12T08:38:22.343 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-04-12 08:31:59.769051815 +0000 2024-04-12T08:38:22.343 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-04-12 08:31:59.769051815 +0000 2024-04-12T08:38:22.344 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-12T08:38:22.408 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-04-12T08:38:22.408 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-04-12T08:38:22.409 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000207964 s, 2.5 MB/s 2024-04-12T08:38:22.410 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-12T08:38:22.465 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_4 2024-04-12T08:38:22.520 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-12T08:38:22.526 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T08:38:22.526 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-04-12T08:38:22.526 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T08:38:22.526 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T08:38:22.526 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-04-12 08:37:58.683906119 +0000 2024-04-12T08:38:22.526 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-04-12 08:32:00.106044427 +0000 2024-04-12T08:38:22.526 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-04-12 08:32:00.106044427 +0000 2024-04-12T08:38:22.526 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-04-12 08:32:00.106044427 +0000 2024-04-12T08:38:22.526 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-12T08:38:22.585 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-04-12T08:38:22.585 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-04-12T08:38:22.585 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000210163 s, 2.4 MB/s 2024-04-12T08:38:22.586 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-12T08:38:22.641 INFO:tasks.cephadm:Deploying osd.0 on smithi090 with /dev/vg_nvme/lv_4... 2024-04-12T08:38:22.641 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-12T08:38:22.897 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:38:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:38:23.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-12T08:38:23.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:38:23.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:23.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:23 smithi114 ceph-mon[33573]: Deploying daemon mgr.x on smithi114 2024-04-12T08:38:23.358 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:23.358 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:23.358 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:23.358 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:23.358 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:23.358 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[40003]: Deploying daemon mgr.x on smithi114 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:38:23.359 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-12T08:38:23.360 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:38:23.360 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:23.360 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:23 smithi090 ceph-mon[29100]: Deploying daemon mgr.x on smithi114 2024-04-12T08:38:24.058 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:23 smithi114 podman[35293]: 2024-04-12 08:38:23.867784011 +0000 UTC m=+0.017709671 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-12T08:38:24.324 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:24 smithi090 ceph-mon[29100]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:24.324 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:24 smithi090 ceph-mon[40003]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:24 smithi114 ceph-mon[33573]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:24.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:24 smithi114 podman[35293]: 2024-04-12 08:38:24.173232328 +0000 UTC m=+0.323157975 container init 1d7b1e033afe78e6763e60ad450a8928704b90baf68e74227d1839f415ed9e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, RELEASE=quincy-d3b5060, org.label-schema.build-date=20240409) 2024-04-12T08:38:24.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:24 smithi114 podman[35293]: 2024-04-12 08:38:24.176140741 +0000 UTC m=+0.326066396 container start 1d7b1e033afe78e6763e60ad450a8928704b90baf68e74227d1839f415ed9e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-12T08:38:24.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:24 smithi114 ceph-mgr[35312]: -- 172.21.15.114:0/4086133172 <== mon.1 v2:172.21.15.114:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5651906809c0 con 0x565190657000 2024-04-12T08:38:24.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:24 smithi114 bash[35293]: 1d7b1e033afe78e6763e60ad450a8928704b90baf68e74227d1839f415ed9e2e 2024-04-12T08:38:24.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:24 smithi114 systemd[1]: Started Ceph mgr.x for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:38:24.982 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-12T08:38:24.983 INFO:teuthology.orchestra.run.smithi090.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-12T08:38:24.983 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10+0 records in 2024-04-12T08:38:24.983 INFO:teuthology.orchestra.run.smithi090.stderr:10+0 records out 2024-04-12T08:38:24.983 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0117116 s, 895 MB/s 2024-04-12T08:38:24.983 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping successful for: 2024-04-12T08:38:25.069 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:24.715+0000 7f81679301c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:38:25.069 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:24.823+0000 7f81679301c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:38:25.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:25.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-mon[33573]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:25.558 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.559 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.559 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.559 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.559 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:25.559 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[29100]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:25.559 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.559 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.559 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.559 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:25.559 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:25.560 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:25 smithi090 ceph-mon[40003]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:25.583 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi090:vg_nvme/lv_4 2024-04-12T08:38:25.812 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:25.473+0000 7f81679301c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:38:25.813 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:25.579+0000 7f81679301c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:38:25.813 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:25.663+0000 7f81679301c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:38:25.860 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:38:26.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:25.812+0000 7f81679301c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:38:26.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:25.892+0000 7f81679301c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:38:26.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:26.061+0000 7f81679301c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:38:26.498 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:26.240+0000 7f81679301c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:38:26.498 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:26.321+0000 7f81679301c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:38:26.498 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:26.412+0000 7f81679301c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:38:26.498 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:26.497+0000 7f81679301c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:38:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:26.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:26.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-mon[33573]: Reconfiguring mgr.y (unknown last config time)... 2024-04-12T08:38:26.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:38:26.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:38:26.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:26.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:26 smithi114 ceph-mon[33573]: Reconfiguring daemon mgr.y on smithi090 2024-04-12T08:38:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[40003]: Reconfiguring mgr.y (unknown last config time)... 2024-04-12T08:38:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:38:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:38:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[40003]: Reconfiguring daemon mgr.y on smithi090 2024-04-12T08:38:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[29100]: Reconfiguring mgr.y (unknown last config time)... 2024-04-12T08:38:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:38:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:38:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:26 smithi090 ceph-mon[29100]: Reconfiguring daemon mgr.y on smithi090 2024-04-12T08:38:27.853 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:27 smithi114 ceph-mon[33573]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:27.853 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:27 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:27.854 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:27.536+0000 7f81679301c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:38:27.854 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:27.664+0000 7f81679301c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:38:27.854 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:27.853+0000 7f81679301c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:38:27.858 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:27 smithi090 ceph-mon[40003]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:27.859 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:27 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:27.859 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:27 smithi090 ceph-mon[29100]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:27.859 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:27 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:28.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:28.084+0000 7f81679301c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:38:28.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:28.258+0000 7f81679301c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:38:28.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:28.434+0000 7f81679301c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:38:29.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:28.689+0000 7f81679301c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:38:29.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:28.773+0000 7f81679301c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:38:29.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:38:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:38:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:29.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:29.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:29.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:29.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:29.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:29.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:29.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:38:29.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:38:29.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:29.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:29.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:29.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:29.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:29.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:29.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:29.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:38:29.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:38:29.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:29.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:29 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:29.158+0000 7f81679301c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:38:30.212 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:29 smithi114 ceph-mon[33573]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:30.212 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:29 smithi114 ceph-mon[33573]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:38:30.212 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:29 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:29.924+0000 7f81679301c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:38:30.237 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:29 smithi090 ceph-mon[29100]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:30.237 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:29 smithi090 ceph-mon[29100]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:38:30.237 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:29 smithi090 ceph-mon[40003]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:30.237 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:29 smithi090 ceph-mon[40003]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:38:30.520 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:30 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:30.211+0000 7f81679301c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:38:30.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:30 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:30.519+0000 7f81679301c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:38:30.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:38:30 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:38:30.710+0000 7f81679301c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:38:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:30 smithi114 ceph-mon[33573]: Standby manager daemon x started 2024-04-12T08:38:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:30 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:38:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:30 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:38:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:30 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:38:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:30 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:38:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:30 smithi090 ceph-mon[29100]: Standby manager daemon x started 2024-04-12T08:38:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:30 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:38:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:30 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:38:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:30 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:38:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:30 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:38:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:30 smithi090 ceph-mon[40003]: Standby manager daemon x started 2024-04-12T08:38:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:30 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:38:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:30 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:38:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:30 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:38:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:30 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/2416578316' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:38:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:31 smithi114 ceph-mon[33573]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:31 smithi114 ceph-mon[33573]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-12T08:38:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:31 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:38:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:31 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1742094480' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3e888cb8-b0cb-4cb8-95ae-7dd29d090968"}]: dispatch 2024-04-12T08:38:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:31 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1742094480' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3e888cb8-b0cb-4cb8-95ae-7dd29d090968"}]': finished 2024-04-12T08:38:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:31 smithi114 ceph-mon[33573]: osdmap e5: 1 total, 0 up, 1 in 2024-04-12T08:38:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:31 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:32.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:31 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2739228744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:38:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[29100]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[29100]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-12T08:38:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:38:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1742094480' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3e888cb8-b0cb-4cb8-95ae-7dd29d090968"}]: dispatch 2024-04-12T08:38:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1742094480' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3e888cb8-b0cb-4cb8-95ae-7dd29d090968"}]': finished 2024-04-12T08:38:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[29100]: osdmap e5: 1 total, 0 up, 1 in 2024-04-12T08:38:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:32.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2739228744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:38:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[40003]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[40003]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-12T08:38:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:38:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1742094480' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3e888cb8-b0cb-4cb8-95ae-7dd29d090968"}]: dispatch 2024-04-12T08:38:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1742094480' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3e888cb8-b0cb-4cb8-95ae-7dd29d090968"}]': finished 2024-04-12T08:38:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[40003]: osdmap e5: 1 total, 0 up, 1 in 2024-04-12T08:38:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:31 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2739228744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:38:34.124 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:34 smithi090 ceph-mon[40003]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:34.124 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:34 smithi090 ceph-mon[29100]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:34 smithi114 ceph-mon[33573]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:36.075 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:36 smithi090 ceph-mon[29100]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:36.075 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:36 smithi090 ceph-mon[40003]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:36.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:36 smithi114 ceph-mon[33573]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:38.263 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:38 smithi090 ceph-mon[40003]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:38.263 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:38 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T08:38:38.263 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:38 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:38.263 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:38 smithi090 ceph-mon[29100]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:38.263 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:38 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T08:38:38.263 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:38 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:38.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:38 smithi114 ceph-mon[33573]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:38.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:38 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T08:38:38.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:38 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:39.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:39 smithi114 ceph-mon[33573]: Deploying daemon osd.0 on smithi090 2024-04-12T08:38:39.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:39 smithi090 ceph-mon[29100]: Deploying daemon osd.0 on smithi090 2024-04-12T08:38:39.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:39 smithi090 ceph-mon[40003]: Deploying daemon osd.0 on smithi090 2024-04-12T08:38:40.167 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:40 smithi090 ceph-mon[29100]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:40.167 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:40 smithi090 ceph-mon[40003]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:40.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:40 smithi114 ceph-mon[33573]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:42.104 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:42 smithi090 ceph-mon[29100]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:42.104 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:42 smithi090 ceph-mon[40003]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:42 smithi114 ceph-mon[33573]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:43.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:43.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:43.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:43.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:43 smithi090 ceph-mon[40003]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:43.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:43.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:43.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:43.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:43 smithi090 ceph-mon[29100]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:43 smithi114 ceph-mon[33573]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:45.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:45.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:45.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[29100]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:45.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:45.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:45.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:45.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[29100]: from='osd.0 [v2:172.21.15.90:6802/937815701,v1:172.21.15.90:6803/937815701]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:38:45.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[29100]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:38:45.795 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:45.795 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:45.796 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[40003]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:45.796 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:45.796 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:45.796 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:45.796 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[40003]: from='osd.0 [v2:172.21.15.90:6802/937815701,v1:172.21.15.90:6803/937815701]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:38:45.796 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:45 smithi090 ceph-mon[40003]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:38:45.863 INFO:teuthology.orchestra.run.smithi090.stdout:Created osd(s) 0 on host 'smithi090' 2024-04-12T08:38:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:45 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:45 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:45 smithi114 ceph-mon[33573]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:45 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:45 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:45 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:45 smithi114 ceph-mon[33573]: from='osd.0 [v2:172.21.15.90:6802/937815701,v1:172.21.15.90:6803/937815701]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:38:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:45 smithi114 ceph-mon[33573]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:38:46.490 DEBUG:teuthology.orchestra.run.smithi090:osd.0> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.0.service 2024-04-12T08:38:46.493 INFO:tasks.cephadm:Deploying osd.1 on smithi090 with /dev/vg_nvme/lv_3... 2024-04-12T08:38:46.493 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-12T08:38:47.117 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:47.117 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:47.117 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:47.117 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[40003]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T08:38:47.117 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[40003]: osdmap e6: 1 total, 0 up, 1 in 2024-04-12T08:38:47.117 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:47.117 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[40003]: from='osd.0 [v2:172.21.15.90:6802/937815701,v1:172.21.15.90:6803/937815701]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:38:47.117 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[40003]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:38:47.117 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:47.117 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:47.118 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:47.118 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[29100]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T08:38:47.118 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[29100]: osdmap e6: 1 total, 0 up, 1 in 2024-04-12T08:38:47.118 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:47.118 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[29100]: from='osd.0 [v2:172.21.15.90:6802/937815701,v1:172.21.15.90:6803/937815701]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:38:47.118 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:46 smithi090 ceph-mon[29100]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:38:47.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:46 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:47.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:46 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:47.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:46 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:38:47.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:46 smithi114 ceph-mon[33573]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T08:38:47.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:46 smithi114 ceph-mon[33573]: osdmap e6: 1 total, 0 up, 1 in 2024-04-12T08:38:47.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:46 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:47.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:46 smithi114 ceph-mon[33573]: from='osd.0 [v2:172.21.15.90:6802/937815701,v1:172.21.15.90:6803/937815701]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:38:47.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:46 smithi114 ceph-mon[33573]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:38:47.382 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[43921]: 2024-04-12T08:38:47.115+0000 7fd5eebee700 -1 osd.0 0 waiting for initial osdmap 2024-04-12T08:38:47.383 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[43921]: 2024-04-12T08:38:47.117+0000 7fd5e81e6700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T08:38:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-mon[29100]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-mon[29100]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:38:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-mon[29100]: osdmap e7: 1 total, 0 up, 1 in 2024-04-12T08:38:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-mon[40003]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-mon[40003]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:38:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-mon[40003]: osdmap e7: 1 total, 0 up, 1 in 2024-04-12T08:38:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:47.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:47 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:47 smithi114 ceph-mon[33573]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:47 smithi114 ceph-mon[33573]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:38:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:47 smithi114 ceph-mon[33573]: osdmap e7: 1 total, 0 up, 1 in 2024-04-12T08:38:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:47 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:47 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:48.134 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:38:49.107 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[40003]: purged_snaps scrub starts 2024-04-12T08:38:49.107 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[40003]: purged_snaps scrub ok 2024-04-12T08:38:49.107 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:49.107 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[40003]: osd.0 [v2:172.21.15.90:6802/937815701,v1:172.21.15.90:6803/937815701] boot 2024-04-12T08:38:49.107 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[40003]: osdmap e8: 1 total, 1 up, 1 in 2024-04-12T08:38:49.107 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:49.107 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[29100]: purged_snaps scrub starts 2024-04-12T08:38:49.107 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[29100]: purged_snaps scrub ok 2024-04-12T08:38:49.107 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:49.107 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[29100]: osd.0 [v2:172.21.15.90:6802/937815701,v1:172.21.15.90:6803/937815701] boot 2024-04-12T08:38:49.108 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[29100]: osdmap e8: 1 total, 1 up, 1 in 2024-04-12T08:38:49.108 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:48 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:48 smithi114 ceph-mon[33573]: purged_snaps scrub starts 2024-04-12T08:38:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:48 smithi114 ceph-mon[33573]: purged_snaps scrub ok 2024-04-12T08:38:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:48 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:48 smithi114 ceph-mon[33573]: osd.0 [v2:172.21.15.90:6802/937815701,v1:172.21.15.90:6803/937815701] boot 2024-04-12T08:38:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:48 smithi114 ceph-mon[33573]: osdmap e8: 1 total, 1 up, 1 in 2024-04-12T08:38:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:48 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:38:50.094 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[29100]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:50.094 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[29100]: osdmap e9: 1 total, 1 up, 1 in 2024-04-12T08:38:50.094 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.094 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.094 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:38:50.094 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.094 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:50.094 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:50.095 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.095 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[40003]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:50.095 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[40003]: osdmap e9: 1 total, 1 up, 1 in 2024-04-12T08:38:50.095 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.095 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.095 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:38:50.095 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.095 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:50.095 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:50.095 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:49 smithi114 ceph-mon[33573]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T08:38:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:49 smithi114 ceph-mon[33573]: osdmap e9: 1 total, 1 up, 1 in 2024-04-12T08:38:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:38:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:38:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:38:50.654 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-12T08:38:50.654 INFO:teuthology.orchestra.run.smithi090.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-12T08:38:50.655 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10+0 records in 2024-04-12T08:38:50.655 INFO:teuthology.orchestra.run.smithi090.stderr:10+0 records out 2024-04-12T08:38:50.655 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114309 s, 917 MB/s 2024-04-12T08:38:50.655 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping successful for: 2024-04-12T08:38:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:50 smithi090 ceph-mon[29100]: Detected new or changed devices on smithi090 2024-04-12T08:38:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:50 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi090 to 16064M 2024-04-12T08:38:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:50 smithi090 ceph-mon[40003]: Detected new or changed devices on smithi090 2024-04-12T08:38:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:50 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi090 to 16064M 2024-04-12T08:38:51.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:50 smithi114 ceph-mon[33573]: Detected new or changed devices on smithi090 2024-04-12T08:38:51.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:50 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi090 to 16064M 2024-04-12T08:38:51.245 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi090:vg_nvme/lv_3 2024-04-12T08:38:51.497 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:38:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:51 smithi090 ceph-mon[29100]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:51 smithi090 ceph-mon[40003]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:51 smithi114 ceph-mon[33573]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:53 smithi090 ceph-mon[29100]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:53 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:38:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:53 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:38:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:53 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:53.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:53 smithi090 ceph-mon[40003]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:53.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:53 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:38:53.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:53 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:38:53.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:53 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:53 smithi114 ceph-mon[33573]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:53 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:38:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:53 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:38:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:53 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:38:55.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:54 smithi114 ceph-mon[33573]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:38:55.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:54 smithi090 ceph-mon[29100]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:38:55.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:54 smithi090 ceph-mon[40003]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:38:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:55 smithi114 ceph-mon[33573]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:55 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2036371012' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "637213e4-7019-4430-ab01-b7196988f82c"}]: dispatch 2024-04-12T08:38:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:55 smithi114 ceph-mon[33573]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "637213e4-7019-4430-ab01-b7196988f82c"}]: dispatch 2024-04-12T08:38:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:55 smithi114 ceph-mon[33573]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "637213e4-7019-4430-ab01-b7196988f82c"}]': finished 2024-04-12T08:38:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:55 smithi114 ceph-mon[33573]: osdmap e10: 2 total, 1 up, 2 in 2024-04-12T08:38:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:55 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:38:56.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[40003]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2036371012' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "637213e4-7019-4430-ab01-b7196988f82c"}]: dispatch 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[40003]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "637213e4-7019-4430-ab01-b7196988f82c"}]: dispatch 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[40003]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "637213e4-7019-4430-ab01-b7196988f82c"}]': finished 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[40003]: osdmap e10: 2 total, 1 up, 2 in 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[29100]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2036371012' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "637213e4-7019-4430-ab01-b7196988f82c"}]: dispatch 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[29100]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "637213e4-7019-4430-ab01-b7196988f82c"}]: dispatch 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[29100]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "637213e4-7019-4430-ab01-b7196988f82c"}]': finished 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[29100]: osdmap e10: 2 total, 1 up, 2 in 2024-04-12T08:38:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:55 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:38:57.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:56 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/212915949' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:38:57.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:56 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/212915949' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:38:57.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:56 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/212915949' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:38:58.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:57 smithi114 ceph-mon[33573]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:58.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:57 smithi090 ceph-mon[40003]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:38:58.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:57 smithi090 ceph-mon[29100]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:38:59 smithi114 ceph-mon[33573]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:00.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:38:59 smithi090 ceph-mon[29100]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:00.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:38:59 smithi090 ceph-mon[40003]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:02.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:01 smithi114 ceph-mon[33573]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:02.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:01 smithi090 ceph-mon[29100]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:02.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:01 smithi090 ceph-mon[40003]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:02 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T08:39:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:02 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:02 smithi114 ceph-mon[33573]: Deploying daemon osd.1 on smithi090 2024-04-12T08:39:03.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:02 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T08:39:03.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:02 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:03.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:02 smithi090 ceph-mon[29100]: Deploying daemon osd.1 on smithi090 2024-04-12T08:39:03.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:02 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T08:39:03.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:02 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:03.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:02 smithi090 ceph-mon[40003]: Deploying daemon osd.1 on smithi090 2024-04-12T08:39:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:03 smithi114 ceph-mon[33573]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:04.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:03 smithi090 ceph-mon[29100]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:04.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:03 smithi090 ceph-mon[40003]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:06.101 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:05 smithi090 ceph-mon[29100]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:06.101 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:05 smithi090 ceph-mon[40003]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:06.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:05 smithi114 ceph-mon[33573]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:08.106 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:07 smithi090 ceph-mon[40003]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:08.106 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:08.106 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:08.106 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:08.106 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:07 smithi090 ceph-mon[29100]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:08.106 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:08.106 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:08.107 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:08.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:07 smithi114 ceph-mon[33573]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:08.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:08.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:08.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:10.063 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:09 smithi090 ceph-mon[40003]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:10.063 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:09 smithi090 ceph-mon[29100]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:10.146 INFO:teuthology.orchestra.run.smithi090.stdout:Created osd(s) 1 on host 'smithi090' 2024-04-12T08:39:10.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:09 smithi114 ceph-mon[33573]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:10.774 DEBUG:teuthology.orchestra.run.smithi090:osd.1> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.1.service 2024-04-12T08:39:10.776 INFO:tasks.cephadm:Deploying osd.2 on smithi090 with /dev/vg_nvme/lv_2... 2024-04-12T08:39:10.776 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-12T08:39:11.082 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[29100]: from='osd.1 [v2:172.21.15.90:6810/2625409332,v1:172.21.15.90:6811/2625409332]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T08:39:11.082 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[29100]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T08:39:11.082 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.082 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.083 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.083 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[40003]: from='osd.1 [v2:172.21.15.90:6810/2625409332,v1:172.21.15.90:6811/2625409332]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T08:39:11.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[40003]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T08:39:11.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:10 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:10 smithi114 ceph-mon[33573]: from='osd.1 [v2:172.21.15.90:6810/2625409332,v1:172.21.15.90:6811/2625409332]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T08:39:11.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:10 smithi114 ceph-mon[33573]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T08:39:11.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:10 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:10 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:10 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:10 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:11.564 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:39:12.239 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[29100]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:12.240 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[29100]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T08:39:12.240 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[29100]: osdmap e11: 2 total, 1 up, 2 in 2024-04-12T08:39:12.240 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:12.240 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[29100]: from='osd.1 [v2:172.21.15.90:6810/2625409332,v1:172.21.15.90:6811/2625409332]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:12.240 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[29100]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:12.240 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[49872]: 2024-04-12T08:39:11.984+0000 7fa20842c700 -1 osd.1 0 waiting for initial osdmap 2024-04-12T08:39:12.240 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[49872]: 2024-04-12T08:39:11.986+0000 7fa201223700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T08:39:12.241 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[40003]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:12.241 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[40003]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T08:39:12.241 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[40003]: osdmap e11: 2 total, 1 up, 2 in 2024-04-12T08:39:12.241 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:12.241 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[40003]: from='osd.1 [v2:172.21.15.90:6810/2625409332,v1:172.21.15.90:6811/2625409332]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:12.241 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:11 smithi090 ceph-mon[40003]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:11 smithi114 ceph-mon[33573]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:11 smithi114 ceph-mon[33573]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T08:39:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:11 smithi114 ceph-mon[33573]: osdmap e11: 2 total, 1 up, 2 in 2024-04-12T08:39:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:11 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:11 smithi114 ceph-mon[33573]: from='osd.1 [v2:172.21.15.90:6810/2625409332,v1:172.21.15.90:6811/2625409332]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:11 smithi114 ceph-mon[33573]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:13.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:13 smithi090 ceph-mon[40003]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:39:13.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:13 smithi090 ceph-mon[40003]: osdmap e12: 2 total, 1 up, 2 in 2024-04-12T08:39:13.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:13 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:13.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:13 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:13.218 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:13 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:13.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:12 smithi090 ceph-mon[29100]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:39:13.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:12 smithi090 ceph-mon[29100]: osdmap e12: 2 total, 1 up, 2 in 2024-04-12T08:39:13.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:13.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:13.219 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:12 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:12 smithi114 ceph-mon[33573]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:39:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:12 smithi114 ceph-mon[33573]: osdmap e12: 2 total, 1 up, 2 in 2024-04-12T08:39:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:12 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:14.001 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-12T08:39:14.001 INFO:teuthology.orchestra.run.smithi090.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-12T08:39:14.001 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10+0 records in 2024-04-12T08:39:14.001 INFO:teuthology.orchestra.run.smithi090.stderr:10+0 records out 2024-04-12T08:39:14.001 INFO:teuthology.orchestra.run.smithi090.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0143265 s, 732 MB/s 2024-04-12T08:39:14.001 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping successful for: 2024-04-12T08:39:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[29100]: purged_snaps scrub starts 2024-04-12T08:39:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[29100]: purged_snaps scrub ok 2024-04-12T08:39:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[29100]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[29100]: osd.1 [v2:172.21.15.90:6810/2625409332,v1:172.21.15.90:6811/2625409332] boot 2024-04-12T08:39:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[29100]: osdmap e13: 2 total, 2 up, 2 in 2024-04-12T08:39:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[40003]: purged_snaps scrub starts 2024-04-12T08:39:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[40003]: purged_snaps scrub ok 2024-04-12T08:39:14.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[40003]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:14.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[40003]: osd.1 [v2:172.21.15.90:6810/2625409332,v1:172.21.15.90:6811/2625409332] boot 2024-04-12T08:39:14.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[40003]: osdmap e13: 2 total, 2 up, 2 in 2024-04-12T08:39:14.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:14 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:14 smithi114 ceph-mon[33573]: purged_snaps scrub starts 2024-04-12T08:39:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:14 smithi114 ceph-mon[33573]: purged_snaps scrub ok 2024-04-12T08:39:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:14 smithi114 ceph-mon[33573]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-12T08:39:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:14 smithi114 ceph-mon[33573]: osd.1 [v2:172.21.15.90:6810/2625409332,v1:172.21.15.90:6811/2625409332] boot 2024-04-12T08:39:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:14 smithi114 ceph-mon[33573]: osdmap e13: 2 total, 2 up, 2 in 2024-04-12T08:39:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:14 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:39:14.702 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi090:vg_nvme/lv_2 2024-04-12T08:39:14.956 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:39:15.331 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:15 smithi090 ceph-mon[40003]: osdmap e14: 2 total, 2 up, 2 in 2024-04-12T08:39:15.331 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:15 smithi090 ceph-mon[29100]: osdmap e14: 2 total, 2 up, 2 in 2024-04-12T08:39:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:15 smithi114 ceph-mon[33573]: osdmap e14: 2 total, 2 up, 2 in 2024-04-12T08:39:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:16.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:16.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:16.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:16.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:16.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:16.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:16 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:16.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:16.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:16.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:16.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:16.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:16.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:16 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:17.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:17 smithi114 ceph-mon[33573]: Detected new or changed devices on smithi090 2024-04-12T08:39:17.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:17 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi090 to 8032M 2024-04-12T08:39:17.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:17 smithi090 ceph-mon[29100]: Detected new or changed devices on smithi090 2024-04-12T08:39:17.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:17 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi090 to 8032M 2024-04-12T08:39:17.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:17 smithi090 ceph-mon[40003]: Detected new or changed devices on smithi090 2024-04-12T08:39:17.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:17 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi090 to 8032M 2024-04-12T08:39:18.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:18 smithi090 ceph-mon[40003]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:18.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:18 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:39:18.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:18 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:39:18.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:18 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:18.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:18 smithi090 ceph-mon[29100]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:18.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:18 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:39:18.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:18 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:39:18.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:18 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:18 smithi114 ceph-mon[33573]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:18 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:39:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:18 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:39:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:18 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:19.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:19 smithi090 ceph-mon[29100]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:39:19.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:19 smithi090 ceph-mon[40003]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:39:19.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:19 smithi114 ceph-mon[33573]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:39:20.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:20 smithi114 ceph-mon[33573]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:20.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:20 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4027349577' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99513c85-6468-4377-85b8-3dc52f0d0f2f"}]: dispatch 2024-04-12T08:39:20.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:20 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4027349577' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "99513c85-6468-4377-85b8-3dc52f0d0f2f"}]': finished 2024-04-12T08:39:20.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:20 smithi114 ceph-mon[33573]: osdmap e15: 3 total, 2 up, 3 in 2024-04-12T08:39:20.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:20 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:20.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:20 smithi090 ceph-mon[29100]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:20.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:20 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4027349577' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99513c85-6468-4377-85b8-3dc52f0d0f2f"}]: dispatch 2024-04-12T08:39:20.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:20 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4027349577' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "99513c85-6468-4377-85b8-3dc52f0d0f2f"}]': finished 2024-04-12T08:39:20.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:20 smithi090 ceph-mon[29100]: osdmap e15: 3 total, 2 up, 3 in 2024-04-12T08:39:20.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:20 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:20.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:20 smithi090 ceph-mon[40003]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:20.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:20 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4027349577' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99513c85-6468-4377-85b8-3dc52f0d0f2f"}]: dispatch 2024-04-12T08:39:20.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:20 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4027349577' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "99513c85-6468-4377-85b8-3dc52f0d0f2f"}]': finished 2024-04-12T08:39:20.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:20 smithi090 ceph-mon[40003]: osdmap e15: 3 total, 2 up, 3 in 2024-04-12T08:39:20.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:20 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:21.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:21 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3633907330' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:39:21.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:21 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3633907330' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:39:21.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:21 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3633907330' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:39:22.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:22 smithi114 ceph-mon[33573]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:22.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:22 smithi090 ceph-mon[29100]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:22.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:22 smithi090 ceph-mon[40003]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:24 smithi090 ceph-mon[40003]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:24 smithi090 ceph-mon[29100]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:24 smithi114 ceph-mon[33573]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:26.364 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:26 smithi090 ceph-mon[40003]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:26.364 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:26 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T08:39:26.364 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:26 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:26.364 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:26 smithi090 ceph-mon[29100]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:26.364 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T08:39:26.364 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:26 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:26 smithi114 ceph-mon[33573]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:26 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T08:39:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:26 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:27.111 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:27 smithi090 ceph-mon[29100]: Deploying daemon osd.2 on smithi090 2024-04-12T08:39:27.111 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:27 smithi090 ceph-mon[40003]: Deploying daemon osd.2 on smithi090 2024-04-12T08:39:27.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:27 smithi114 ceph-mon[33573]: Deploying daemon osd.2 on smithi090 2024-04-12T08:39:28.320 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:28 smithi090 ceph-mon[40003]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:28.320 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:28 smithi090 ceph-mon[29100]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:28 smithi114 ceph-mon[33573]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:30.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:30 smithi114 ceph-mon[33573]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:30.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:30 smithi090 ceph-mon[29100]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:30.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:30 smithi090 ceph-mon[40003]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:32 smithi114 ceph-mon[33573]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:32 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:32 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:32 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:32 smithi090 ceph-mon[29100]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:32 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:32 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:32 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:32.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:32 smithi090 ceph-mon[40003]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:32.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:32 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:32.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:32 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:32.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:32 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:34.154 INFO:teuthology.orchestra.run.smithi090.stdout:Created osd(s) 2 on host 'smithi090' 2024-04-12T08:39:34.192 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:34 smithi090 ceph-mon[29100]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:34.193 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:34 smithi090 ceph-mon[40003]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:34 smithi114 ceph-mon[33573]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:34.798 DEBUG:teuthology.orchestra.run.smithi090:osd.2> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.2.service 2024-04-12T08:39:34.800 INFO:tasks.cephadm:Deploying osd.3 on smithi090 with /dev/vg_nvme/lv_1... 2024-04-12T08:39:34.800 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-12T08:39:35.116 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:39:35.402 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.402 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.402 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[40003]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T08:39:35.402 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[40003]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:35.402 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.402 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.403 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.403 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.403 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[29100]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T08:39:35.403 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[29100]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:35.403 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.403 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:35 smithi114 ceph-mon[33573]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T08:39:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:35 smithi114 ceph-mon[33573]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:36.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:36 smithi090 ceph-mon[29100]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T08:39:36.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:36 smithi090 ceph-mon[29100]: osdmap e16: 3 total, 2 up, 3 in 2024-04-12T08:39:36.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:36 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:36.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:36 smithi090 ceph-mon[29100]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:36.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:36 smithi090 ceph-mon[40003]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T08:39:36.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:36 smithi090 ceph-mon[40003]: osdmap e16: 3 total, 2 up, 3 in 2024-04-12T08:39:36.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:36 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:36.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:36 smithi090 ceph-mon[40003]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:36.424 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 08:39:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[55718]: 2024-04-12T08:39:36.145+0000 7ff83b08d700 -1 osd.2 0 waiting for initial osdmap 2024-04-12T08:39:36.424 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 08:39:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[55718]: 2024-04-12T08:39:36.149+0000 7ff835e88700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T08:39:36.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:36 smithi114 ceph-mon[33573]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T08:39:36.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:36 smithi114 ceph-mon[33573]: osdmap e16: 3 total, 2 up, 3 in 2024-04-12T08:39:36.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:36 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:36.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:36 smithi114 ceph-mon[33573]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:37.083 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-12T08:39:37.084 INFO:teuthology.orchestra.run.smithi090.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-12T08:39:37.084 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10+0 records in 2024-04-12T08:39:37.084 INFO:teuthology.orchestra.run.smithi090.stderr:10+0 records out 2024-04-12T08:39:37.084 INFO:teuthology.orchestra.run.smithi090.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0130552 s, 803 MB/s 2024-04-12T08:39:37.084 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping successful for: 2024-04-12T08:39:37.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:37 smithi090 ceph-mon[29100]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:39:37.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:37 smithi090 ceph-mon[29100]: osdmap e17: 3 total, 2 up, 3 in 2024-04-12T08:39:37.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:37 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:37.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:37 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:37.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:37 smithi090 ceph-mon[29100]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:37.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:37 smithi090 ceph-mon[40003]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:39:37.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:37 smithi090 ceph-mon[40003]: osdmap e17: 3 total, 2 up, 3 in 2024-04-12T08:39:37.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:37 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:37.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:37 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:37.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:37 smithi090 ceph-mon[40003]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:37.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:37 smithi114 ceph-mon[33573]: from='osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:39:37.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:37 smithi114 ceph-mon[33573]: osdmap e17: 3 total, 2 up, 3 in 2024-04-12T08:39:37.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:37 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:37.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:37 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:37.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:37 smithi114 ceph-mon[33573]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-12T08:39:38.303 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:38 smithi090 ceph-mon[40003]: purged_snaps scrub starts 2024-04-12T08:39:38.304 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:38 smithi090 ceph-mon[40003]: purged_snaps scrub ok 2024-04-12T08:39:38.304 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:38 smithi090 ceph-mon[40003]: osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287] boot 2024-04-12T08:39:38.304 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:38 smithi090 ceph-mon[40003]: osdmap e18: 3 total, 3 up, 3 in 2024-04-12T08:39:38.304 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:38 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:38.304 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:38 smithi090 ceph-mon[29100]: purged_snaps scrub starts 2024-04-12T08:39:38.304 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:38 smithi090 ceph-mon[29100]: purged_snaps scrub ok 2024-04-12T08:39:38.304 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:38 smithi090 ceph-mon[29100]: osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287] boot 2024-04-12T08:39:38.304 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:38 smithi090 ceph-mon[29100]: osdmap e18: 3 total, 3 up, 3 in 2024-04-12T08:39:38.304 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:38 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:38.518 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:38 smithi114 ceph-mon[33573]: purged_snaps scrub starts 2024-04-12T08:39:38.518 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:38 smithi114 ceph-mon[33573]: purged_snaps scrub ok 2024-04-12T08:39:38.518 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:38 smithi114 ceph-mon[33573]: osd.2 [v2:172.21.15.90:6818/3723833287,v1:172.21.15.90:6819/3723833287] boot 2024-04-12T08:39:38.518 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:38 smithi114 ceph-mon[33573]: osdmap e18: 3 total, 3 up, 3 in 2024-04-12T08:39:38.519 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:38 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:39:38.704 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi090:vg_nvme/lv_1 2024-04-12T08:39:39.043 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:39:39.311 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:39 smithi090 ceph-mon[40003]: osdmap e19: 3 total, 3 up, 3 in 2024-04-12T08:39:39.311 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:39 smithi090 ceph-mon[40003]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:39.311 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:39 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-12T08:39:39.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:39 smithi090 ceph-mon[29100]: osdmap e19: 3 total, 3 up, 3 in 2024-04-12T08:39:39.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:39 smithi090 ceph-mon[29100]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:39.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:39 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-12T08:39:39.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:39 smithi114 ceph-mon[33573]: osdmap e19: 3 total, 3 up, 3 in 2024-04-12T08:39:39.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:39 smithi114 ceph-mon[33573]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:39.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:39 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-12T08:39:40.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:40 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-12T08:39:40.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:40 smithi090 ceph-mon[29100]: osdmap e20: 3 total, 3 up, 3 in 2024-04-12T08:39:40.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:40 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:39:40.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:40 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-12T08:39:40.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:40 smithi090 ceph-mon[40003]: osdmap e20: 3 total, 3 up, 3 in 2024-04-12T08:39:40.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:40 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:39:40.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:40 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-12T08:39:40.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:40 smithi114 ceph-mon[33573]: osdmap e20: 3 total, 3 up, 3 in 2024-04-12T08:39:40.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:40 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:39:40.831 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:39:40 smithi090 sudo[59419]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T08:39:40.831 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:39:40 smithi090 sudo[59419]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:40.831 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:39:40 smithi090 sudo[59419]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:40.831 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:39:40 smithi090 sudo[59419]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:41.138 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:39:40 smithi090 sudo[59422]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T08:39:41.138 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:39:40 smithi090 sudo[59422]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:41.138 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:39:40 smithi090 sudo[59422]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:41.138 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:39:40 smithi090 sudo[59422]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:41.420 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59496]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T08:39:41.420 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59496]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:41.420 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59496]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:41.420 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59496]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:41.420 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:39:41.420 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: osdmap e21: 3 total, 3 up, 3 in 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: Detected new or changed devices on smithi090 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi090 to 5354M 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: osdmap e21: 3 total, 3 up, 3 in 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: Detected new or changed devices on smithi090 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:41.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi090 to 5354M 2024-04-12T08:39:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: osdmap e21: 3 total, 3 up, 3 in 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: Detected new or changed devices on smithi090 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi090 to 5354M 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:41.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:41.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:41.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:41.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:39:41.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:41.672 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59521]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T08:39:41.672 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59521]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:41.672 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59521]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:41.673 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59521]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:41.940 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59591]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T08:39:41.941 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59591]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:41.941 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59591]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:41.941 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59591]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:42.203 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 sudo[59656]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T08:39:42.203 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 sudo[59656]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:42.203 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 sudo[59656]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:42.203 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 sudo[59656]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:42.203 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59648]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T08:39:42.203 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59648]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:42.204 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59648]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:42.204 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 08:39:41 smithi090 sudo[59648]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:42.455 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 sudo[59672]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T08:39:42.455 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 sudo[59672]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:42.455 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 sudo[59672]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:42.455 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 sudo[59672]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:42.455 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 ceph-mon[29100]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-12T08:39:42.455 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 ceph-mon[29100]: osdmap e22: 3 total, 3 up, 3 in 2024-04-12T08:39:42.455 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:39:42.455 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:39:42.455 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:42 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:42.708 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 ceph-mon[33573]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-12T08:39:42.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 ceph-mon[33573]: osdmap e22: 3 total, 3 up, 3 in 2024-04-12T08:39:42.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:39:42.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:39:42.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:42.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 sudo[36011]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T08:39:42.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 sudo[36011]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:42.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 sudo[36011]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:42.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 sudo[36011]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:42 smithi090 ceph-mon[40003]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-12T08:39:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:42 smithi090 ceph-mon[40003]: osdmap e22: 3 total, 3 up, 3 in 2024-04-12T08:39:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:42 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:39:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:42 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:39:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:42 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 sudo[36014]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T08:39:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 sudo[36014]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 sudo[36014]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:42 smithi114 sudo[36014]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:43.235 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 sudo[59701]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T08:39:43.235 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 sudo[59701]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:43.235 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 sudo[59701]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:43.235 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 sudo[59701]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:43.492 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 sudo[59710]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T08:39:43.492 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 sudo[59710]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T08:39:43.492 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 sudo[59710]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T08:39:43.492 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 sudo[59710]: pam_unix(sudo:session): session closed for user root 2024-04-12T08:39:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:39:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T08:39:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:39:43.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:39:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:39:43.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T08:39:43.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:39:43.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:39:43.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:39:43.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:43.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T08:39:43.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:43.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:39:43.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:39:43.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:39:44.824 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[40003]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T08:39:44.824 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[40003]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T08:39:44.824 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2199330253' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "afd01784-8987-4954-9be5-57d950beb611"}]: dispatch 2024-04-12T08:39:44.824 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2199330253' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "afd01784-8987-4954-9be5-57d950beb611"}]': finished 2024-04-12T08:39:44.824 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[40003]: osdmap e23: 4 total, 3 up, 4 in 2024-04-12T08:39:44.825 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:39:44.825 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[29100]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T08:39:44.825 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[29100]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T08:39:44.825 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2199330253' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "afd01784-8987-4954-9be5-57d950beb611"}]: dispatch 2024-04-12T08:39:44.825 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2199330253' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "afd01784-8987-4954-9be5-57d950beb611"}]': finished 2024-04-12T08:39:44.825 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[29100]: osdmap e23: 4 total, 3 up, 4 in 2024-04-12T08:39:44.825 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:44 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:39:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:44 smithi114 ceph-mon[33573]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T08:39:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:44 smithi114 ceph-mon[33573]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T08:39:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:44 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2199330253' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "afd01784-8987-4954-9be5-57d950beb611"}]: dispatch 2024-04-12T08:39:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:44 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2199330253' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "afd01784-8987-4954-9be5-57d950beb611"}]': finished 2024-04-12T08:39:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:44 smithi114 ceph-mon[33573]: osdmap e23: 4 total, 3 up, 4 in 2024-04-12T08:39:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:44 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:39:45.834 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:45 smithi090 ceph-mon[40003]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:45.834 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:45 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4043253543' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:39:45.834 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:45 smithi090 ceph-mon[29100]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:45.834 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:45 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4043253543' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:39:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:45 smithi114 ceph-mon[33573]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:45 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4043253543' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:39:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:47 smithi090 ceph-mon[40003]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:47 smithi090 ceph-mon[29100]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:47 smithi114 ceph-mon[33573]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:49 smithi090 ceph-mon[29100]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:49 smithi090 ceph-mon[40003]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:49 smithi114 ceph-mon[33573]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:50 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T08:39:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:50 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:50 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T08:39:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:50 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:51.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:50 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T08:39:51.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:50 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:39:51.910 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:51 smithi090 ceph-mon[29100]: Deploying daemon osd.3 on smithi090 2024-04-12T08:39:51.910 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:51 smithi090 ceph-mon[29100]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:51.910 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:51 smithi090 ceph-mon[40003]: Deploying daemon osd.3 on smithi090 2024-04-12T08:39:51.910 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:51 smithi090 ceph-mon[40003]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:51 smithi114 ceph-mon[33573]: Deploying daemon osd.3 on smithi090 2024-04-12T08:39:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:51 smithi114 ceph-mon[33573]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:53 smithi090 ceph-mon[29100]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:53.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:53 smithi090 ceph-mon[40003]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:53 smithi114 ceph-mon[33573]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:55.659 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:55 smithi090 ceph-mon[29100]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:55.659 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:55 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:55.659 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:55 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:55.659 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:55 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:55.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:55 smithi090 ceph-mon[40003]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:55.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:55 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:55.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:55 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:55.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:55 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:55 smithi114 ceph-mon[33573]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:55 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:55 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:55 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:39:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:57 smithi090 ceph-mon[29100]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:57.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:57 smithi090 ceph-mon[40003]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:58.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:57 smithi114 ceph-mon[33573]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:58 smithi090 ceph-mon[29100]: from='osd.3 [v2:172.21.15.90:6826/1425929386,v1:172.21.15.90:6827/1425929386]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T08:39:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:58 smithi090 ceph-mon[29100]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T08:39:58.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:58 smithi090 ceph-mon[40003]: from='osd.3 [v2:172.21.15.90:6826/1425929386,v1:172.21.15.90:6827/1425929386]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T08:39:58.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:58 smithi090 ceph-mon[40003]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T08:39:59.107 INFO:teuthology.orchestra.run.smithi090.stdout:Created osd(s) 3 on host 'smithi090' 2024-04-12T08:39:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:58 smithi114 ceph-mon[33573]: from='osd.3 [v2:172.21.15.90:6826/1425929386,v1:172.21.15.90:6827/1425929386]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T08:39:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:58 smithi114 ceph-mon[33573]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T08:39:59.739 DEBUG:teuthology.orchestra.run.smithi090:osd.3> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.3.service 2024-04-12T08:39:59.742 INFO:tasks.cephadm:Deploying osd.4 on smithi114 with /dev/vg_nvme/lv_4... 2024-04-12T08:39:59.742 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-12T08:39:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[40003]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[40003]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T08:39:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[40003]: osdmap e24: 4 total, 3 up, 4 in 2024-04-12T08:39:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:39:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[40003]: from='osd.3 [v2:172.21.15.90:6826/1425929386,v1:172.21.15.90:6827/1425929386]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[40003]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[29100]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[29100]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[29100]: osdmap e24: 4 total, 3 up, 4 in 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[29100]: from='osd.3 [v2:172.21.15.90:6826/1425929386,v1:172.21.15.90:6827/1425929386]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[29100]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:59.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:59.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:59.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:39:59 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:39:59.998 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:40:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:59 smithi114 ceph-mon[33573]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:40:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:59 smithi114 ceph-mon[33573]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T08:40:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:59 smithi114 ceph-mon[33573]: osdmap e24: 4 total, 3 up, 4 in 2024-04-12T08:40:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:59 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:59 smithi114 ceph-mon[33573]: from='osd.3 [v2:172.21.15.90:6826/1425929386,v1:172.21.15.90:6827/1425929386]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:40:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:59 smithi114 ceph-mon[33573]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:40:00.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:59 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:00.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:59 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:00.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:59 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:00.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:39:59 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:00 smithi090 ceph-mon[29100]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:40:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:00 smithi090 ceph-mon[29100]: osdmap e25: 4 total, 3 up, 4 in 2024-04-12T08:40:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:00 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:00 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:00 smithi090 ceph-mon[29100]: overall HEALTH_OK 2024-04-12T08:40:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:00 smithi090 ceph-mon[40003]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:40:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:00 smithi090 ceph-mon[40003]: osdmap e25: 4 total, 3 up, 4 in 2024-04-12T08:40:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:00 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:00 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:00.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:00 smithi090 ceph-mon[40003]: overall HEALTH_OK 2024-04-12T08:40:01.026 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:00 smithi114 ceph-mon[33573]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-12T08:40:01.026 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:00 smithi114 ceph-mon[33573]: osdmap e25: 4 total, 3 up, 4 in 2024-04-12T08:40:01.026 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:00 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:01.026 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:00 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:01.026 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:00 smithi114 ceph-mon[33573]: overall HEALTH_OK 2024-04-12T08:40:01.968 INFO:teuthology.orchestra.run.smithi114.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-12T08:40:01.968 INFO:teuthology.orchestra.run.smithi114.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-12T08:40:01.968 INFO:teuthology.orchestra.run.smithi114.stderr: stderr: 10+0 records in 2024-04-12T08:40:01.968 INFO:teuthology.orchestra.run.smithi114.stderr:10+0 records out 2024-04-12T08:40:01.968 INFO:teuthology.orchestra.run.smithi114.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114275 s, 918 MB/s 2024-04-12T08:40:01.968 INFO:teuthology.orchestra.run.smithi114.stderr:--> Zapping successful for: 2024-04-12T08:40:01.969 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:01 smithi114 ceph-mon[33573]: purged_snaps scrub starts 2024-04-12T08:40:01.969 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:01 smithi114 ceph-mon[33573]: purged_snaps scrub ok 2024-04-12T08:40:01.969 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:01 smithi114 ceph-mon[33573]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:40:01.969 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:01 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:01.969 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:01 smithi114 ceph-mon[33573]: osd.3 [v2:172.21.15.90:6826/1425929386,v1:172.21.15.90:6827/1425929386] boot 2024-04-12T08:40:01.969 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:01 smithi114 ceph-mon[33573]: osdmap e26: 4 total, 4 up, 4 in 2024-04-12T08:40:01.969 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:01 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[29100]: purged_snaps scrub starts 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[29100]: purged_snaps scrub ok 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[29100]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[29100]: osd.3 [v2:172.21.15.90:6826/1425929386,v1:172.21.15.90:6827/1425929386] boot 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[29100]: osdmap e26: 4 total, 4 up, 4 in 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[40003]: purged_snaps scrub starts 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[40003]: purged_snaps scrub ok 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[40003]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[40003]: osd.3 [v2:172.21.15.90:6826/1425929386,v1:172.21.15.90:6827/1425929386] boot 2024-04-12T08:40:02.045 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[40003]: osdmap e26: 4 total, 4 up, 4 in 2024-04-12T08:40:02.046 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:01 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:40:02.556 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi114:vg_nvme/lv_4 2024-04-12T08:40:02.814 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:40:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:02 smithi114 ceph-mon[33573]: osdmap e27: 4 total, 4 up, 4 in 2024-04-12T08:40:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:02 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:02 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:02 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:02 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:02.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:02 smithi090 ceph-mon[29100]: osdmap e27: 4 total, 4 up, 4 in 2024-04-12T08:40:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:02 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:02 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:02 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:02 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:02 smithi090 ceph-mon[40003]: osdmap e27: 4 total, 4 up, 4 in 2024-04-12T08:40:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:02 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:02 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:02 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:02 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: Detected new or changed devices on smithi090 2024-04-12T08:40:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi090 to 4016M 2024-04-12T08:40:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:03.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:03.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:03.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:03.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:03.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:03.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:03.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:03 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: Detected new or changed devices on smithi090 2024-04-12T08:40:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi090 to 4016M 2024-04-12T08:40:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: Detected new or changed devices on smithi090 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi090 to 4016M 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:04.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:04.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:04.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:04.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:04.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:04.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:04.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:03 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:04.968 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:40:04.968 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:40:04.968 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:05.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:04 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:40:05.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:04 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:40:05.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:04 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:40:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:40:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:05.840 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:05 smithi114 ceph-mon[33573]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:40:05.840 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:05 smithi114 ceph-mon[33573]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:06.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:05 smithi090 ceph-mon[29100]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:40:06.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:05 smithi090 ceph-mon[29100]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:06.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:05 smithi090 ceph-mon[40003]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:40:06.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:05 smithi090 ceph-mon[40003]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:07 smithi114 ceph-mon[33573]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:07 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/659518107' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "596f9e87-d99e-423c-902f-f14c5dcdcf36"}]: dispatch 2024-04-12T08:40:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:07 smithi114 ceph-mon[33573]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "596f9e87-d99e-423c-902f-f14c5dcdcf36"}]: dispatch 2024-04-12T08:40:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:07 smithi114 ceph-mon[33573]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "596f9e87-d99e-423c-902f-f14c5dcdcf36"}]': finished 2024-04-12T08:40:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:07 smithi114 ceph-mon[33573]: osdmap e28: 5 total, 4 up, 5 in 2024-04-12T08:40:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:07 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/4039744469' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:40:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[29100]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/659518107' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "596f9e87-d99e-423c-902f-f14c5dcdcf36"}]: dispatch 2024-04-12T08:40:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[29100]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "596f9e87-d99e-423c-902f-f14c5dcdcf36"}]: dispatch 2024-04-12T08:40:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[29100]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "596f9e87-d99e-423c-902f-f14c5dcdcf36"}]': finished 2024-04-12T08:40:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[29100]: osdmap e28: 5 total, 4 up, 5 in 2024-04-12T08:40:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/4039744469' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:40:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[40003]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/659518107' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "596f9e87-d99e-423c-902f-f14c5dcdcf36"}]: dispatch 2024-04-12T08:40:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[40003]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "596f9e87-d99e-423c-902f-f14c5dcdcf36"}]: dispatch 2024-04-12T08:40:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[40003]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "596f9e87-d99e-423c-902f-f14c5dcdcf36"}]': finished 2024-04-12T08:40:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[40003]: osdmap e28: 5 total, 4 up, 5 in 2024-04-12T08:40:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:07 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/4039744469' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:40:10.006 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:09 smithi114 ceph-mon[33573]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:10.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:09 smithi090 ceph-mon[40003]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:10.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:09 smithi090 ceph-mon[29100]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:11.919 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:11 smithi114 ceph-mon[33573]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:11 smithi090 ceph-mon[29100]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:11 smithi090 ceph-mon[40003]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:13 smithi114 ceph-mon[33573]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:13 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T08:40:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:13 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:13 smithi114 ceph-mon[33573]: Deploying daemon osd.4 on smithi114 2024-04-12T08:40:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:13 smithi090 ceph-mon[29100]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:13 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T08:40:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:13 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:13 smithi090 ceph-mon[29100]: Deploying daemon osd.4 on smithi114 2024-04-12T08:40:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:13 smithi090 ceph-mon[40003]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:13 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T08:40:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:13 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:13 smithi090 ceph-mon[40003]: Deploying daemon osd.4 on smithi114 2024-04-12T08:40:15.975 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:15 smithi090 ceph-mon[40003]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:15.976 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:15 smithi090 ceph-mon[29100]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:16.059 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:15 smithi114 ceph-mon[33573]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:17 smithi114 ceph-mon[33573]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:18.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:17 smithi090 ceph-mon[40003]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:18.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:17 smithi090 ceph-mon[29100]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:19.018 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:18 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:19.018 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:18 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:19.018 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:18 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:19.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:18 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:19.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:18 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:19.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:18 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:19.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:18 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:19.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:18 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:19.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:18 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:19.967 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:19 smithi114 ceph-mon[33573]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:20.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:19 smithi090 ceph-mon[29100]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:20.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:19 smithi090 ceph-mon[40003]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:20.666 INFO:teuthology.orchestra.run.smithi114.stdout:Created osd(s) 4 on host 'smithi114' 2024-04-12T08:40:21.230 DEBUG:teuthology.orchestra.run.smithi114:osd.4> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.4.service 2024-04-12T08:40:21.233 INFO:tasks.cephadm:Deploying osd.5 on smithi114 with /dev/vg_nvme/lv_3... 2024-04-12T08:40:21.233 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-12T08:40:21.537 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:40:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[29100]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[29100]: from='osd.4 [v2:172.21.15.114:6800/1802592095,v1:172.21.15.114:6801/1802592095]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T08:40:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[29100]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T08:40:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[40003]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:21.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[40003]: from='osd.4 [v2:172.21.15.114:6800/1802592095,v1:172.21.15.114:6801/1802592095]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T08:40:21.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[40003]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T08:40:21.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:21 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.942 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:21 smithi114 ceph-mon[33573]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:21.943 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:21 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.943 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:21 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.943 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:21 smithi114 ceph-mon[33573]: from='osd.4 [v2:172.21.15.114:6800/1802592095,v1:172.21.15.114:6801/1802592095]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T08:40:21.943 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:21 smithi114 ceph-mon[33573]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T08:40:21.943 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:21 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:21.943 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:21 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:22.808 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:22 smithi114 ceph-mon[33573]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T08:40:22.808 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:22 smithi114 ceph-mon[33573]: osdmap e29: 5 total, 4 up, 5 in 2024-04-12T08:40:22.808 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:22 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:22.808 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:22 smithi114 ceph-mon[33573]: from='osd.4 [v2:172.21.15.114:6800/1802592095,v1:172.21.15.114:6801/1802592095]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:22.808 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:22 smithi114 ceph-mon[33573]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:23.117 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 08:40:22 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T08:40:22.805+0000 7f190b4a0700 -1 osd.4 0 waiting for initial osdmap 2024-04-12T08:40:23.117 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 08:40:22 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T08:40:22.810+0000 7f1905a9a700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T08:40:23.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:22 smithi090 ceph-mon[29100]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T08:40:23.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:22 smithi090 ceph-mon[29100]: osdmap e29: 5 total, 4 up, 5 in 2024-04-12T08:40:23.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:22 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:23.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:22 smithi090 ceph-mon[29100]: from='osd.4 [v2:172.21.15.114:6800/1802592095,v1:172.21.15.114:6801/1802592095]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:23.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:22 smithi090 ceph-mon[29100]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:23.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:22 smithi090 ceph-mon[40003]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T08:40:23.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:22 smithi090 ceph-mon[40003]: osdmap e29: 5 total, 4 up, 5 in 2024-04-12T08:40:23.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:22 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:23.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:22 smithi090 ceph-mon[40003]: from='osd.4 [v2:172.21.15.114:6800/1802592095,v1:172.21.15.114:6801/1802592095]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:23.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:22 smithi090 ceph-mon[40003]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:24.050 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:23 smithi114 ceph-mon[33573]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:24.050 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:23 smithi114 ceph-mon[33573]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:40:24.050 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:23 smithi114 ceph-mon[33573]: osdmap e30: 5 total, 4 up, 5 in 2024-04-12T08:40:24.050 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:24.050 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:24.139 INFO:teuthology.orchestra.run.smithi114.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-12T08:40:24.139 INFO:teuthology.orchestra.run.smithi114.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-12T08:40:24.139 INFO:teuthology.orchestra.run.smithi114.stderr: stderr: 10+0 records in 2024-04-12T08:40:24.139 INFO:teuthology.orchestra.run.smithi114.stderr:10+0 records out 2024-04-12T08:40:24.139 INFO:teuthology.orchestra.run.smithi114.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123514 s, 849 MB/s 2024-04-12T08:40:24.140 INFO:teuthology.orchestra.run.smithi114.stderr:--> Zapping successful for: 2024-04-12T08:40:24.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:23 smithi090 ceph-mon[29100]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:23 smithi090 ceph-mon[29100]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:40:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:23 smithi090 ceph-mon[29100]: osdmap e30: 5 total, 4 up, 5 in 2024-04-12T08:40:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:23 smithi090 ceph-mon[40003]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-12T08:40:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:23 smithi090 ceph-mon[40003]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:40:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:23 smithi090 ceph-mon[40003]: osdmap e30: 5 total, 4 up, 5 in 2024-04-12T08:40:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:24.919 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:24 smithi114 ceph-mon[33573]: purged_snaps scrub starts 2024-04-12T08:40:24.919 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:24 smithi114 ceph-mon[33573]: purged_snaps scrub ok 2024-04-12T08:40:24.919 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:24 smithi114 ceph-mon[33573]: osd.4 [v2:172.21.15.114:6800/1802592095,v1:172.21.15.114:6801/1802592095] boot 2024-04-12T08:40:24.919 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:24 smithi114 ceph-mon[33573]: osdmap e31: 5 total, 5 up, 5 in 2024-04-12T08:40:24.920 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:24 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:24.921 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi114:vg_nvme/lv_3 2024-04-12T08:40:25.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:24 smithi090 ceph-mon[29100]: purged_snaps scrub starts 2024-04-12T08:40:25.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:24 smithi090 ceph-mon[29100]: purged_snaps scrub ok 2024-04-12T08:40:25.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:24 smithi090 ceph-mon[29100]: osd.4 [v2:172.21.15.114:6800/1802592095,v1:172.21.15.114:6801/1802592095] boot 2024-04-12T08:40:25.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:24 smithi090 ceph-mon[29100]: osdmap e31: 5 total, 5 up, 5 in 2024-04-12T08:40:25.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:24 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:25.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:24 smithi090 ceph-mon[40003]: purged_snaps scrub starts 2024-04-12T08:40:25.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:24 smithi090 ceph-mon[40003]: purged_snaps scrub ok 2024-04-12T08:40:25.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:24 smithi090 ceph-mon[40003]: osd.4 [v2:172.21.15.114:6800/1802592095,v1:172.21.15.114:6801/1802592095] boot 2024-04-12T08:40:25.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:24 smithi090 ceph-mon[40003]: osdmap e31: 5 total, 5 up, 5 in 2024-04-12T08:40:25.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:24 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:40:25.499 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:40:25.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:25.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: osdmap e32: 5 total, 5 up, 5 in 2024-04-12T08:40:25.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:25.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:25.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:25.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:25.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:25.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:25.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:25.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:25.905 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:25.905 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:25.905 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:25 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: osdmap e32: 5 total, 5 up, 5 in 2024-04-12T08:40:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: osdmap e32: 5 total, 5 up, 5 in 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:26.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:26.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:26.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:26.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:26.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:26.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:25 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:27.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:26 smithi114 ceph-mon[33573]: Detected new or changed devices on smithi114 2024-04-12T08:40:27.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:26 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi114 to 17088M 2024-04-12T08:40:27.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:26 smithi114 ceph-mon[33573]: osdmap e33: 5 total, 5 up, 5 in 2024-04-12T08:40:27.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:26 smithi114 ceph-mon[33573]: osdmap e34: 5 total, 5 up, 5 in 2024-04-12T08:40:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:26 smithi090 ceph-mon[29100]: Detected new or changed devices on smithi114 2024-04-12T08:40:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:26 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi114 to 17088M 2024-04-12T08:40:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:26 smithi090 ceph-mon[29100]: osdmap e33: 5 total, 5 up, 5 in 2024-04-12T08:40:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:26 smithi090 ceph-mon[29100]: osdmap e34: 5 total, 5 up, 5 in 2024-04-12T08:40:27.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:26 smithi090 ceph-mon[40003]: Detected new or changed devices on smithi114 2024-04-12T08:40:27.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:26 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi114 to 17088M 2024-04-12T08:40:27.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:26 smithi090 ceph-mon[40003]: osdmap e33: 5 total, 5 up, 5 in 2024-04-12T08:40:27.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:26 smithi090 ceph-mon[40003]: osdmap e34: 5 total, 5 up, 5 in 2024-04-12T08:40:27.842 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:27 smithi114 ceph-mon[33573]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:27.842 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:27 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:40:27.843 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:27 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:40:27.843 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:27 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:27.843 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:27 smithi114 ceph-mon[33573]: osdmap e35: 5 total, 5 up, 5 in 2024-04-12T08:40:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:27 smithi090 ceph-mon[29100]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:27 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:40:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:27 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:40:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:27 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:27 smithi090 ceph-mon[29100]: osdmap e35: 5 total, 5 up, 5 in 2024-04-12T08:40:28.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:27 smithi090 ceph-mon[40003]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:28.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:27 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:40:28.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:27 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:40:28.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:27 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:28.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:27 smithi090 ceph-mon[40003]: osdmap e35: 5 total, 5 up, 5 in 2024-04-12T08:40:28.995 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:28 smithi114 ceph-mon[33573]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:40:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:28 smithi090 ceph-mon[29100]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:40:29.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:28 smithi090 ceph-mon[40003]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:40:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:29 smithi114 ceph-mon[33573]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:29 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/981940416' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2374027-89a9-4670-9593-3ddf9393b286"}]: dispatch 2024-04-12T08:40:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:29 smithi114 ceph-mon[33573]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2374027-89a9-4670-9593-3ddf9393b286"}]: dispatch 2024-04-12T08:40:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:29 smithi114 ceph-mon[33573]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b2374027-89a9-4670-9593-3ddf9393b286"}]': finished 2024-04-12T08:40:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:29 smithi114 ceph-mon[33573]: osdmap e36: 6 total, 5 up, 6 in 2024-04-12T08:40:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:29 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[29100]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/981940416' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2374027-89a9-4670-9593-3ddf9393b286"}]: dispatch 2024-04-12T08:40:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[29100]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2374027-89a9-4670-9593-3ddf9393b286"}]: dispatch 2024-04-12T08:40:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[29100]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b2374027-89a9-4670-9593-3ddf9393b286"}]': finished 2024-04-12T08:40:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[29100]: osdmap e36: 6 total, 5 up, 6 in 2024-04-12T08:40:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:30.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[40003]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:30.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/981940416' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2374027-89a9-4670-9593-3ddf9393b286"}]: dispatch 2024-04-12T08:40:30.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[40003]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2374027-89a9-4670-9593-3ddf9393b286"}]: dispatch 2024-04-12T08:40:30.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[40003]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b2374027-89a9-4670-9593-3ddf9393b286"}]': finished 2024-04-12T08:40:30.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[40003]: osdmap e36: 6 total, 5 up, 6 in 2024-04-12T08:40:30.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:29 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:30 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/1433823022' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:40:31.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:30 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/1433823022' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:40:31.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:30 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/1433823022' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:40:32.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:31 smithi114 ceph-mon[33573]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-04-12T08:40:32.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:31 smithi090 ceph-mon[29100]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-04-12T08:40:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:31 smithi090 ceph-mon[40003]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-04-12T08:40:34.056 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:33 smithi114 ceph-mon[33573]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-04-12T08:40:34.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:33 smithi090 ceph-mon[29100]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-04-12T08:40:34.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:33 smithi090 ceph-mon[40003]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-04-12T08:40:36.066 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:35 smithi114 ceph-mon[33573]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T08:40:36.066 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T08:40:36.066 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:36.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:35 smithi090 ceph-mon[29100]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T08:40:36.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T08:40:36.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:36.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:35 smithi090 ceph-mon[40003]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T08:40:36.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T08:40:36.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:37.114 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:36 smithi114 ceph-mon[33573]: Deploying daemon osd.5 on smithi114 2024-04-12T08:40:37.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:36 smithi090 ceph-mon[29100]: Deploying daemon osd.5 on smithi114 2024-04-12T08:40:37.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:36 smithi090 ceph-mon[40003]: Deploying daemon osd.5 on smithi114 2024-04-12T08:40:38.116 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:37 smithi114 ceph-mon[33573]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-12T08:40:38.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:37 smithi090 ceph-mon[29100]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-12T08:40:38.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:37 smithi090 ceph-mon[40003]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-12T08:40:40.110 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:39 smithi114 ceph-mon[33573]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T08:40:40.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:39 smithi090 ceph-mon[29100]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T08:40:40.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:39 smithi090 ceph-mon[40003]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T08:40:41.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:41.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:41.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:41 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:41.770 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:41 smithi114 ceph-mon[33573]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:41.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:41.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:41.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:41 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:41.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:41 smithi090 ceph-mon[29100]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:41.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:41.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:41.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:41 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:41.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:41 smithi090 ceph-mon[40003]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:43.805 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:43 smithi114 ceph-mon[33573]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:43.805 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:43.805 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:43.806 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:43.806 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:43.806 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:43.806 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:43 smithi114 ceph-mon[33573]: from='osd.5 [v2:172.21.15.114:6808/117630080,v1:172.21.15.114:6809/117630080]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T08:40:43.806 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:43 smithi114 ceph-mon[33573]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T08:40:43.892 INFO:teuthology.orchestra.run.smithi114.stdout:Created osd(s) 5 on host 'smithi114' 2024-04-12T08:40:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[29100]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[29100]: from='osd.5 [v2:172.21.15.114:6808/117630080,v1:172.21.15.114:6809/117630080]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T08:40:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[29100]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T08:40:44.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[40003]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:44.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:44.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:44.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:44.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:44.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:44.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[40003]: from='osd.5 [v2:172.21.15.114:6808/117630080,v1:172.21.15.114:6809/117630080]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T08:40:44.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:43 smithi090 ceph-mon[40003]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T08:40:44.483 DEBUG:teuthology.orchestra.run.smithi114:osd.5> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.5.service 2024-04-12T08:40:44.485 INFO:tasks.cephadm:Deploying osd.6 on smithi114 with /dev/vg_nvme/lv_2... 2024-04-12T08:40:44.486 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-12T08:40:45.082 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[29100]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[29100]: osdmap e37: 6 total, 5 up, 6 in 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[29100]: from='osd.5 [v2:172.21.15.114:6808/117630080,v1:172.21.15.114:6809/117630080]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[29100]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[40003]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[40003]: osdmap e37: 6 total, 5 up, 6 in 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[40003]: from='osd.5 [v2:172.21.15.114:6808/117630080,v1:172.21.15.114:6809/117630080]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:45.083 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[40003]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:45.084 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:45.084 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:45.084 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:44 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:44 smithi114 ceph-mon[33573]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T08:40:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:44 smithi114 ceph-mon[33573]: osdmap e37: 6 total, 5 up, 6 in 2024-04-12T08:40:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:44 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:44 smithi114 ceph-mon[33573]: from='osd.5 [v2:172.21.15.114:6808/117630080,v1:172.21.15.114:6809/117630080]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:44 smithi114 ceph-mon[33573]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:40:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:44 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:44 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:45.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:44 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:40:45.119 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 08:40:44 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T08:40:44.775+0000 7f5d14e22700 -1 osd.5 0 waiting for initial osdmap 2024-04-12T08:40:45.119 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 08:40:44 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T08:40:44.780+0000 7f5d0dc19700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T08:40:45.534 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:40:45.993 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:45 smithi114 ceph-mon[33573]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:45.993 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:45 smithi114 ceph-mon[33573]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:40:45.993 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:45 smithi114 ceph-mon[33573]: osdmap e38: 6 total, 5 up, 6 in 2024-04-12T08:40:45.994 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:45 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:45.994 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:45 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:45 smithi090 ceph-mon[29100]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:45 smithi090 ceph-mon[29100]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:40:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:45 smithi090 ceph-mon[29100]: osdmap e38: 6 total, 5 up, 6 in 2024-04-12T08:40:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:45 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:45 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:46.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:45 smithi090 ceph-mon[40003]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:46.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:45 smithi090 ceph-mon[40003]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:40:46.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:45 smithi090 ceph-mon[40003]: osdmap e38: 6 total, 5 up, 6 in 2024-04-12T08:40:46.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:45 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:46.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:45 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:47.008 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:46 smithi114 ceph-mon[33573]: purged_snaps scrub starts 2024-04-12T08:40:47.008 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:46 smithi114 ceph-mon[33573]: purged_snaps scrub ok 2024-04-12T08:40:47.008 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:46 smithi114 ceph-mon[33573]: osd.5 [v2:172.21.15.114:6808/117630080,v1:172.21.15.114:6809/117630080] boot 2024-04-12T08:40:47.008 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:46 smithi114 ceph-mon[33573]: osdmap e39: 6 total, 6 up, 6 in 2024-04-12T08:40:47.009 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:46 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:47.009 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:46 smithi114 ceph-mon[33573]: osdmap e40: 6 total, 6 up, 6 in 2024-04-12T08:40:47.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[29100]: purged_snaps scrub starts 2024-04-12T08:40:47.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[29100]: purged_snaps scrub ok 2024-04-12T08:40:47.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[29100]: osd.5 [v2:172.21.15.114:6808/117630080,v1:172.21.15.114:6809/117630080] boot 2024-04-12T08:40:47.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[29100]: osdmap e39: 6 total, 6 up, 6 in 2024-04-12T08:40:47.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:47.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[29100]: osdmap e40: 6 total, 6 up, 6 in 2024-04-12T08:40:47.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[40003]: purged_snaps scrub starts 2024-04-12T08:40:47.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[40003]: purged_snaps scrub ok 2024-04-12T08:40:47.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[40003]: osd.5 [v2:172.21.15.114:6808/117630080,v1:172.21.15.114:6809/117630080] boot 2024-04-12T08:40:47.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[40003]: osdmap e39: 6 total, 6 up, 6 in 2024-04-12T08:40:47.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:40:47.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:46 smithi090 ceph-mon[40003]: osdmap e40: 6 total, 6 up, 6 in 2024-04-12T08:40:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:47 smithi114 ceph-mon[33573]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:47 smithi114 ceph-mon[33573]: osdmap e41: 6 total, 6 up, 6 in 2024-04-12T08:40:47.992 INFO:teuthology.orchestra.run.smithi114.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-12T08:40:47.992 INFO:teuthology.orchestra.run.smithi114.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-12T08:40:47.993 INFO:teuthology.orchestra.run.smithi114.stderr: stderr: 10+0 records in 2024-04-12T08:40:47.993 INFO:teuthology.orchestra.run.smithi114.stderr:10+0 records out 2024-04-12T08:40:47.993 INFO:teuthology.orchestra.run.smithi114.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0116053 s, 904 MB/s 2024-04-12T08:40:47.993 INFO:teuthology.orchestra.run.smithi114.stderr:--> Zapping successful for: 2024-04-12T08:40:48.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:47 smithi090 ceph-mon[29100]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:48.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:47 smithi090 ceph-mon[29100]: osdmap e41: 6 total, 6 up, 6 in 2024-04-12T08:40:48.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:47 smithi090 ceph-mon[40003]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-12T08:40:48.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:47 smithi090 ceph-mon[40003]: osdmap e41: 6 total, 6 up, 6 in 2024-04-12T08:40:48.610 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi114:vg_nvme/lv_2 2024-04-12T08:40:48.859 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:40:49.281 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: Detected new or changed devices on smithi114 2024-04-12T08:40:49.281 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.281 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.281 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:49.281 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:49.281 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi114 to 8544M 2024-04-12T08:40:49.281 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.281 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:49.282 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:49.282 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.282 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:49 smithi114 ceph-mon[33573]: osdmap e42: 6 total, 6 up, 6 in 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: Detected new or changed devices on smithi114 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi114 to 8544M 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[40003]: osdmap e42: 6 total, 6 up, 6 in 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: Detected new or changed devices on smithi114 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi114 to 8544M 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:40:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:49 smithi090 ceph-mon[29100]: osdmap e42: 6 total, 6 up, 6 in 2024-04-12T08:40:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:50 smithi114 ceph-mon[33573]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:50 smithi114 ceph-mon[33573]: osdmap e43: 6 total, 6 up, 6 in 2024-04-12T08:40:50.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:50 smithi090 ceph-mon[29100]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:50.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:50 smithi090 ceph-mon[29100]: osdmap e43: 6 total, 6 up, 6 in 2024-04-12T08:40:50.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:50 smithi090 ceph-mon[40003]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:50.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:50 smithi090 ceph-mon[40003]: osdmap e43: 6 total, 6 up, 6 in 2024-04-12T08:40:51.234 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:51 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:40:51.235 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:51 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:40:51.235 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:51 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:51.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:51 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:40:51.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:51 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:40:51.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:51 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:51.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:40:51.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:40:51.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:52 smithi114 ceph-mon[33573]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:40:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:52 smithi114 ceph-mon[33573]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:52.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:52 smithi090 ceph-mon[40003]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:40:52.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:52 smithi090 ceph-mon[40003]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:52.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:52 smithi090 ceph-mon[29100]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:40:52.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:52 smithi090 ceph-mon[29100]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:53.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:53 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/4102322209' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "487cdd03-1123-4386-9237-97b2bc6dd92f"}]: dispatch 2024-04-12T08:40:53.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:53 smithi114 ceph-mon[33573]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "487cdd03-1123-4386-9237-97b2bc6dd92f"}]: dispatch 2024-04-12T08:40:53.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:53 smithi114 ceph-mon[33573]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "487cdd03-1123-4386-9237-97b2bc6dd92f"}]': finished 2024-04-12T08:40:53.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:53 smithi114 ceph-mon[33573]: osdmap e44: 7 total, 6 up, 7 in 2024-04-12T08:40:53.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:53 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:40:53.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:53 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/4102322209' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "487cdd03-1123-4386-9237-97b2bc6dd92f"}]: dispatch 2024-04-12T08:40:53.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:53 smithi090 ceph-mon[29100]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "487cdd03-1123-4386-9237-97b2bc6dd92f"}]: dispatch 2024-04-12T08:40:53.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:53 smithi090 ceph-mon[29100]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "487cdd03-1123-4386-9237-97b2bc6dd92f"}]': finished 2024-04-12T08:40:53.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:53 smithi090 ceph-mon[29100]: osdmap e44: 7 total, 6 up, 7 in 2024-04-12T08:40:53.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:53 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:40:53.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:53 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/4102322209' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "487cdd03-1123-4386-9237-97b2bc6dd92f"}]: dispatch 2024-04-12T08:40:53.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:53 smithi090 ceph-mon[40003]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "487cdd03-1123-4386-9237-97b2bc6dd92f"}]: dispatch 2024-04-12T08:40:53.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:53 smithi090 ceph-mon[40003]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "487cdd03-1123-4386-9237-97b2bc6dd92f"}]': finished 2024-04-12T08:40:53.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:53 smithi090 ceph-mon[40003]: osdmap e44: 7 total, 6 up, 7 in 2024-04-12T08:40:53.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:53 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:40:54.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:54 smithi114 ceph-mon[33573]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:54.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:54 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/2775998743' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:40:54.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:54 smithi090 ceph-mon[29100]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:54.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:54 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/2775998743' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:40:54.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:54 smithi090 ceph-mon[40003]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:54.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:54 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/2775998743' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:40:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:56 smithi114 ceph-mon[33573]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:56.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:56 smithi090 ceph-mon[29100]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:56.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:56 smithi090 ceph-mon[40003]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:58.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:58 smithi114 ceph-mon[33573]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:58.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:58 smithi090 ceph-mon[29100]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:58 smithi090 ceph-mon[40003]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:40:59.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:59 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T08:40:59.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:40:59 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:59.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:59 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T08:40:59.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:40:59 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:40:59.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:59 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T08:40:59.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:40:59 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:00.324 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:00 smithi114 ceph-mon[33573]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:00.324 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:00 smithi114 ceph-mon[33573]: Deploying daemon osd.6 on smithi114 2024-04-12T08:41:00.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:00 smithi090 ceph-mon[29100]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:00.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:00 smithi090 ceph-mon[29100]: Deploying daemon osd.6 on smithi114 2024-04-12T08:41:00.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:00 smithi090 ceph-mon[40003]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:00.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:00 smithi090 ceph-mon[40003]: Deploying daemon osd.6 on smithi114 2024-04-12T08:41:02.350 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:02 smithi114 ceph-mon[33573]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:02.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:02 smithi090 ceph-mon[29100]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:02.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:02 smithi090 ceph-mon[40003]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:04.335 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:04 smithi114 ceph-mon[33573]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:04.335 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:04.335 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:04.335 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:04 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:04.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:04 smithi090 ceph-mon[40003]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:04.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:04 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:04.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:04 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:04.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:04 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:04 smithi090 ceph-mon[29100]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:04 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:06.279 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:06 smithi114 ceph-mon[33573]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:06.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:06 smithi090 ceph-mon[40003]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:06.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:06 smithi090 ceph-mon[29100]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:07.241 INFO:teuthology.orchestra.run.smithi114.stdout:Created osd(s) 6 on host 'smithi114' 2024-04-12T08:41:07.629 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.629 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.629 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:07.629 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:07.629 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:07.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: from='osd.6 [v2:172.21.15.114:6816/2139728372,v1:172.21.15.114:6817/2139728372]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T08:41:07.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T08:41:07.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:07 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: from='osd.6 [v2:172.21.15.114:6816/2139728372,v1:172.21.15.114:6817/2139728372]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: from='osd.6 [v2:172.21.15.114:6816/2139728372,v1:172.21.15.114:6817/2139728372]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:07.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:07 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:07.961 DEBUG:teuthology.orchestra.run.smithi114:osd.6> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.6.service 2024-04-12T08:41:07.964 INFO:tasks.cephadm:Deploying osd.7 on smithi114 with /dev/vg_nvme/lv_1... 2024-04-12T08:41:07.964 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-12T08:41:08.608 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:41:08.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:08 smithi114 ceph-mon[33573]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T08:41:08.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:08 smithi114 ceph-mon[33573]: osdmap e45: 7 total, 6 up, 7 in 2024-04-12T08:41:08.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:08 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:08.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:08 smithi114 ceph-mon[33573]: from='osd.6 [v2:172.21.15.114:6816/2139728372,v1:172.21.15.114:6817/2139728372]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:08.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:08 smithi114 ceph-mon[33573]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:08.617 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 08:41:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T08:41:08.395+0000 7f2cb0ca8700 -1 osd.6 0 waiting for initial osdmap 2024-04-12T08:41:08.617 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 08:41:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T08:41:08.401+0000 7f2cad2a6700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T08:41:08.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:08 smithi090 ceph-mon[29100]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T08:41:08.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:08 smithi090 ceph-mon[29100]: osdmap e45: 7 total, 6 up, 7 in 2024-04-12T08:41:08.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:08 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:08.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:08 smithi090 ceph-mon[29100]: from='osd.6 [v2:172.21.15.114:6816/2139728372,v1:172.21.15.114:6817/2139728372]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:08.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:08 smithi090 ceph-mon[29100]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:08.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:08 smithi090 ceph-mon[40003]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T08:41:08.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:08 smithi090 ceph-mon[40003]: osdmap e45: 7 total, 6 up, 7 in 2024-04-12T08:41:08.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:08 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:08.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:08 smithi090 ceph-mon[40003]: from='osd.6 [v2:172.21.15.114:6816/2139728372,v1:172.21.15.114:6817/2139728372]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:08.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:08 smithi090 ceph-mon[40003]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:09.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:09 smithi090 ceph-mon[40003]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:41:09.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:09 smithi090 ceph-mon[40003]: osdmap e46: 7 total, 6 up, 7 in 2024-04-12T08:41:09.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:09 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:09.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:09 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:09.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:09 smithi090 ceph-mon[40003]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:09.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:09 smithi090 ceph-mon[29100]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:41:09.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:09 smithi090 ceph-mon[29100]: osdmap e46: 7 total, 6 up, 7 in 2024-04-12T08:41:09.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:09 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:09.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:09 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:09.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:09 smithi090 ceph-mon[29100]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:09.698 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:09 smithi114 ceph-mon[33573]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:41:09.698 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:09 smithi114 ceph-mon[33573]: osdmap e46: 7 total, 6 up, 7 in 2024-04-12T08:41:09.698 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:09 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:09.698 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:09 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:09.698 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:09 smithi114 ceph-mon[33573]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-12T08:41:10.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:10 smithi090 ceph-mon[40003]: purged_snaps scrub starts 2024-04-12T08:41:10.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:10 smithi090 ceph-mon[40003]: purged_snaps scrub ok 2024-04-12T08:41:10.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:10 smithi090 ceph-mon[40003]: osd.6 [v2:172.21.15.114:6816/2139728372,v1:172.21.15.114:6817/2139728372] boot 2024-04-12T08:41:10.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:10 smithi090 ceph-mon[40003]: osdmap e47: 7 total, 7 up, 7 in 2024-04-12T08:41:10.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:10 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:10.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:10 smithi090 ceph-mon[29100]: purged_snaps scrub starts 2024-04-12T08:41:10.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:10 smithi090 ceph-mon[29100]: purged_snaps scrub ok 2024-04-12T08:41:10.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:10 smithi090 ceph-mon[29100]: osd.6 [v2:172.21.15.114:6816/2139728372,v1:172.21.15.114:6817/2139728372] boot 2024-04-12T08:41:10.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:10 smithi090 ceph-mon[29100]: osdmap e47: 7 total, 7 up, 7 in 2024-04-12T08:41:10.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:10 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:10.708 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:10 smithi114 ceph-mon[33573]: purged_snaps scrub starts 2024-04-12T08:41:10.708 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:10 smithi114 ceph-mon[33573]: purged_snaps scrub ok 2024-04-12T08:41:10.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:10 smithi114 ceph-mon[33573]: osd.6 [v2:172.21.15.114:6816/2139728372,v1:172.21.15.114:6817/2139728372] boot 2024-04-12T08:41:10.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:10 smithi114 ceph-mon[33573]: osdmap e47: 7 total, 7 up, 7 in 2024-04-12T08:41:10.709 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:10 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:41:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: osdmap e48: 7 total, 7 up, 7 in 2024-04-12T08:41:11.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: pgmap v150: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:11.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:11.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:11.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:11.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:11.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:11.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: osdmap e48: 7 total, 7 up, 7 in 2024-04-12T08:41:11.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: pgmap v150: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:11.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:11.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:11.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:11.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:11.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:11.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:11 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.806 INFO:teuthology.orchestra.run.smithi114.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-12T08:41:11.806 INFO:teuthology.orchestra.run.smithi114.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-12T08:41:11.806 INFO:teuthology.orchestra.run.smithi114.stderr: stderr: 10+0 records in 2024-04-12T08:41:11.806 INFO:teuthology.orchestra.run.smithi114.stderr:10+0 records out 2024-04-12T08:41:11.806 INFO:teuthology.orchestra.run.smithi114.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0127182 s, 824 MB/s 2024-04-12T08:41:11.806 INFO:teuthology.orchestra.run.smithi114.stderr:--> Zapping successful for: 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: osdmap e48: 7 total, 7 up, 7 in 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: pgmap v150: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:11.807 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:11 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:12.406 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi114:vg_nvme/lv_1 2024-04-12T08:41:12.655 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:41:12.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:12 smithi090 ceph-mon[40003]: Detected new or changed devices on smithi114 2024-04-12T08:41:12.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:12 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi114 to 5696M 2024-04-12T08:41:12.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:12 smithi090 ceph-mon[40003]: osdmap e49: 7 total, 7 up, 7 in 2024-04-12T08:41:12.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:12 smithi090 ceph-mon[29100]: Detected new or changed devices on smithi114 2024-04-12T08:41:12.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:12 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi114 to 5696M 2024-04-12T08:41:12.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:12 smithi090 ceph-mon[29100]: osdmap e49: 7 total, 7 up, 7 in 2024-04-12T08:41:12.749 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:12 smithi114 ceph-mon[33573]: Detected new or changed devices on smithi114 2024-04-12T08:41:12.749 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:12 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi114 to 5696M 2024-04-12T08:41:12.749 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:12 smithi114 ceph-mon[33573]: osdmap e49: 7 total, 7 up, 7 in 2024-04-12T08:41:13.679 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:13 smithi114 ceph-mon[33573]: osdmap e50: 7 total, 7 up, 7 in 2024-04-12T08:41:13.679 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:13 smithi114 ceph-mon[33573]: pgmap v153: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:13.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:13 smithi090 ceph-mon[40003]: osdmap e50: 7 total, 7 up, 7 in 2024-04-12T08:41:13.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:13 smithi090 ceph-mon[40003]: pgmap v153: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:13.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:13 smithi090 ceph-mon[29100]: osdmap e50: 7 total, 7 up, 7 in 2024-04-12T08:41:13.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:13 smithi090 ceph-mon[29100]: pgmap v153: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:14.758 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:14 smithi114 ceph-mon[33573]: osdmap e51: 7 total, 7 up, 7 in 2024-04-12T08:41:14.759 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:14 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:41:14.759 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:14 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:41:14.759 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:14 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:14 smithi090 ceph-mon[29100]: osdmap e51: 7 total, 7 up, 7 in 2024-04-12T08:41:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:14 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:41:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:14 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:41:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:14 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:14.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:14 smithi090 ceph-mon[40003]: osdmap e51: 7 total, 7 up, 7 in 2024-04-12T08:41:14.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:14 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T08:41:14.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:14 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T08:41:14.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:14 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:15.747 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:15 smithi114 ceph-mon[33573]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:15.748 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:15 smithi114 ceph-mon[33573]: pgmap v155: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:15.824 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:15 smithi090 ceph-mon[29100]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:15.824 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:15 smithi090 ceph-mon[29100]: pgmap v155: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:15.824 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:15 smithi090 ceph-mon[40003]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:15.825 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:15 smithi090 ceph-mon[40003]: pgmap v155: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[29100]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T08:41:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/2837285596' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d46acbb-c271-4f1f-b5ba-090a44443308"}]: dispatch 2024-04-12T08:41:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[29100]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d46acbb-c271-4f1f-b5ba-090a44443308"}]: dispatch 2024-04-12T08:41:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[29100]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0d46acbb-c271-4f1f-b5ba-090a44443308"}]': finished 2024-04-12T08:41:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[29100]: osdmap e52: 8 total, 7 up, 8 in 2024-04-12T08:41:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/3149637462' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:41:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[40003]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T08:41:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/2837285596' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d46acbb-c271-4f1f-b5ba-090a44443308"}]: dispatch 2024-04-12T08:41:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[40003]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d46acbb-c271-4f1f-b5ba-090a44443308"}]: dispatch 2024-04-12T08:41:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[40003]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0d46acbb-c271-4f1f-b5ba-090a44443308"}]': finished 2024-04-12T08:41:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[40003]: osdmap e52: 8 total, 7 up, 8 in 2024-04-12T08:41:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:17 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/3149637462' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:41:18.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:17 smithi114 ceph-mon[33573]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T08:41:18.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:17 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/2837285596' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d46acbb-c271-4f1f-b5ba-090a44443308"}]: dispatch 2024-04-12T08:41:18.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:17 smithi114 ceph-mon[33573]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d46acbb-c271-4f1f-b5ba-090a44443308"}]: dispatch 2024-04-12T08:41:18.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:17 smithi114 ceph-mon[33573]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0d46acbb-c271-4f1f-b5ba-090a44443308"}]': finished 2024-04-12T08:41:18.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:17 smithi114 ceph-mon[33573]: osdmap e52: 8 total, 7 up, 8 in 2024-04-12T08:41:18.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:17 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:18.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:17 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/3149637462' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T08:41:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:19 smithi090 ceph-mon[29100]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-12T08:41:19.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:19 smithi090 ceph-mon[40003]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-12T08:41:19.997 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:19 smithi114 ceph-mon[33573]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-12T08:41:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:21 smithi090 ceph-mon[29100]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T08:41:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:21 smithi090 ceph-mon[40003]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T08:41:22.008 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:21 smithi114 ceph-mon[33573]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T08:41:23.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:23 smithi114 ceph-mon[33573]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-12T08:41:23.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T08:41:23.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:23 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:23.904 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:23 smithi114 ceph-mon[33573]: Deploying daemon osd.7 on smithi114 2024-04-12T08:41:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:23 smithi090 ceph-mon[29100]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-12T08:41:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T08:41:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:23 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:23 smithi090 ceph-mon[29100]: Deploying daemon osd.7 on smithi114 2024-04-12T08:41:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:23 smithi090 ceph-mon[40003]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-12T08:41:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T08:41:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:23 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:23 smithi090 ceph-mon[40003]: Deploying daemon osd.7 on smithi114 2024-04-12T08:41:25.799 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:25 smithi114 ceph-mon[33573]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T08:41:26.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:25 smithi090 ceph-mon[40003]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T08:41:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:25 smithi090 ceph-mon[29100]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T08:41:27.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:27 smithi114 ceph-mon[33573]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:28.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:27 smithi090 ceph-mon[29100]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:28.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:27 smithi090 ceph-mon[40003]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:28 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:28 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:29.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:29.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:29.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:28 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:29 smithi114 ceph-mon[33573]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:29 smithi090 ceph-mon[29100]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:30.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:29 smithi090 ceph-mon[40003]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:31.373 INFO:teuthology.orchestra.run.smithi114.stdout:Created osd(s) 7 on host 'smithi114' 2024-04-12T08:41:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: from='osd.7 [v2:172.21.15.114:6824/3563377268,v1:172.21.15.114:6825/3563377268]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T08:41:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T08:41:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:31 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: from='osd.7 [v2:172.21.15.114:6824/3563377268,v1:172.21.15.114:6825/3563377268]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:31.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:31.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: from='osd.7 [v2:172.21.15.114:6824/3563377268,v1:172.21.15.114:6825/3563377268]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T08:41:31.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T08:41:31.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:31.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:31.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:31.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:31 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:31.997 DEBUG:teuthology.orchestra.run.smithi114:osd.7> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.7.service 2024-04-12T08:41:32.000 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-12T08:41:32.000 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd stat -f json 2024-04-12T08:41:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:41:32.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:32 smithi090 ceph-mon[29100]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T08:41:32.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:32 smithi090 ceph-mon[29100]: osdmap e53: 8 total, 7 up, 8 in 2024-04-12T08:41:32.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:32 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:32.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:32 smithi090 ceph-mon[29100]: from='osd.7 [v2:172.21.15.114:6824/3563377268,v1:172.21.15.114:6825/3563377268]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:32.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:32 smithi090 ceph-mon[29100]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:32 smithi090 ceph-mon[40003]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T08:41:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:32 smithi090 ceph-mon[40003]: osdmap e53: 8 total, 7 up, 8 in 2024-04-12T08:41:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:32 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:32 smithi090 ceph-mon[40003]: from='osd.7 [v2:172.21.15.114:6824/3563377268,v1:172.21.15.114:6825/3563377268]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:32.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:32 smithi090 ceph-mon[40003]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:32 smithi114 ceph-mon[33573]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T08:41:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:32 smithi114 ceph-mon[33573]: osdmap e53: 8 total, 7 up, 8 in 2024-04-12T08:41:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:32 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:32 smithi114 ceph-mon[33573]: from='osd.7 [v2:172.21.15.114:6824/3563377268,v1:172.21.15.114:6825/3563377268]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:32 smithi114 ceph-mon[33573]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-04-12T08:41:32.868 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 08:41:32 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T08:41:32.618+0000 7f74b9661700 -1 osd.7 0 waiting for initial osdmap 2024-04-12T08:41:32.868 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 08:41:32 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T08:41:32.624+0000 7f74b445c700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T08:41:33.737 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:33 smithi090 ceph-mon[40003]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:41:33.737 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:33 smithi090 ceph-mon[40003]: osdmap e54: 8 total, 7 up, 8 in 2024-04-12T08:41:33.737 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:33 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:33.737 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:33 smithi090 ceph-mon[40003]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:33.738 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:33 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:33.738 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:33 smithi090 ceph-mon[29100]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:41:33.738 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:33 smithi090 ceph-mon[29100]: osdmap e54: 8 total, 7 up, 8 in 2024-04-12T08:41:33.738 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:33 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:33.738 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:33 smithi090 ceph-mon[29100]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:33.738 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:33 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:33.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:33 smithi114 ceph-mon[33573]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-04-12T08:41:33.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:33 smithi114 ceph-mon[33573]: osdmap e54: 8 total, 7 up, 8 in 2024-04-12T08:41:33.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:33 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:33.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:33 smithi114 ceph-mon[33573]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-12T08:41:33.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:33 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:34.154 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:41:34.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[29100]: purged_snaps scrub starts 2024-04-12T08:41:34.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[29100]: purged_snaps scrub ok 2024-04-12T08:41:34.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[29100]: osd.7 [v2:172.21.15.114:6824/3563377268,v1:172.21.15.114:6825/3563377268] boot 2024-04-12T08:41:34.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[29100]: osdmap e55: 8 total, 8 up, 8 in 2024-04-12T08:41:34.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:34.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3974332592' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-12T08:41:34.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[40003]: purged_snaps scrub starts 2024-04-12T08:41:34.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[40003]: purged_snaps scrub ok 2024-04-12T08:41:34.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[40003]: osd.7 [v2:172.21.15.114:6824/3563377268,v1:172.21.15.114:6825/3563377268] boot 2024-04-12T08:41:34.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[40003]: osdmap e55: 8 total, 8 up, 8 in 2024-04-12T08:41:34.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:34.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:34 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3974332592' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-12T08:41:34.771 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1712911293,"num_in_osds":8,"osd_in_since":1712911276,"num_remapped_pgs":1} 2024-04-12T08:41:34.772 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T08:41:34.937 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:34 smithi114 ceph-mon[33573]: purged_snaps scrub starts 2024-04-12T08:41:34.937 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:34 smithi114 ceph-mon[33573]: purged_snaps scrub ok 2024-04-12T08:41:34.937 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:34 smithi114 ceph-mon[33573]: osd.7 [v2:172.21.15.114:6824/3563377268,v1:172.21.15.114:6825/3563377268] boot 2024-04-12T08:41:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:34 smithi114 ceph-mon[33573]: osdmap e55: 8 total, 8 up, 8 in 2024-04-12T08:41:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:34 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:41:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:34 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3974332592' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-12T08:41:35.033 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:41:35.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:35.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: osdmap e56: 8 total, 8 up, 8 in 2024-04-12T08:41:35.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: Detected new or changed devices on smithi114 2024-04-12T08:41:35.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:35.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:35.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi114 to 4272M 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: osdmap e56: 8 total, 8 up, 8 in 2024-04-12T08:41:35.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: Detected new or changed devices on smithi114 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi114 to 4272M 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:35.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:35 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: osdmap e56: 8 total, 8 up, 8 in 2024-04-12T08:41:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: Detected new or changed devices on smithi114 2024-04-12T08:41:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:36.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:36.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:36.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:41:36.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi114 to 4272M 2024-04-12T08:41:36.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:36.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:36.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:36.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:35 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:36.780 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:41:36.780 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":58,"fsid":"96735376-f8a7-11ee-bc8c-c7b262605968","created":"2024-04-12T08:36:35.370883+0000","modified":"2024-04-12T08:41:36.313305+0000","last_up_change":"2024-04-12T08:41:33.609734+0000","last_in_change":"2024-04-12T08:41:16.832300+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T08:39:38.877058+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"3e888cb8-b0cb-4cb8-95ae-7dd29d090968","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6803","nonce":937815701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6805","nonce":937815701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6809","nonce":937815701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6807","nonce":937815701}]},"public_addr":"172.21.15.90:6803/937815701","cluster_addr":"172.21.15.90:6805/937815701","heartbeat_back_addr":"172.21.15.90:6809/937815701","heartbeat_front_addr":"172.21.15.90:6807/937815701","state":["exists","up"]},{"osd":1,"uuid":"637213e4-7019-4430-ab01-b7196988f82c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6811","nonce":2625409332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6813","nonce":2625409332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6817","nonce":2625409332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6815","nonce":2625409332}]},"public_addr":"172.21.15.90:6811/2625409332","cluster_addr":"172.21.15.90:6813/2625409332","heartbeat_back_addr":"172.21.15.90:6817/2625409332","heartbeat_front_addr":"172.21.15.90:6815/2625409332","state":["exists","up"]},{"osd":2,"uuid":"99513c85-6468-4377-85b8-3dc52f0d0f2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6819","nonce":3723833287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6821","nonce":3723833287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6825","nonce":3723833287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6823","nonce":3723833287}]},"public_addr":"172.21.15.90:6819/3723833287","cluster_addr":"172.21.15.90:6821/3723833287","heartbeat_back_addr":"172.21.15.90:6825/3723833287","heartbeat_front_addr":"172.21.15.90:6823/3723833287","state":["exists","up"]},{"osd":3,"uuid":"afd01784-8987-4954-9be5-57d950beb611","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6827","nonce":1425929386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6829","nonce":1425929386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6833","nonce":1425929386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6831","nonce":1425929386}]},"public_addr":"172.21.15.90:6827/1425929386","cluster_addr":"172.21.15.90:6829/1425929386","heartbeat_back_addr":"172.21.15.90:6833/1425929386","heartbeat_front_addr":"172.21.15.90:6831/1425929386","state":["exists","up"]},{"osd":4,"uuid":"596f9e87-d99e-423c-902f-f14c5dcdcf36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6801","nonce":1802592095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6803","nonce":1802592095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6807","nonce":1802592095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6805","nonce":1802592095}]},"public_addr":"172.21.15.114:6801/1802592095","cluster_addr":"172.21.15.114:6803/1802592095","heartbeat_back_addr":"172.21.15.114:6807/1802592095","heartbeat_front_addr":"172.21.15.114:6805/1802592095","state":["exists","up"]},{"osd":5,"uuid":"b2374027-89a9-4670-9593-3ddf9393b286","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6809","nonce":117630080}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6811","nonce":117630080}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6815","nonce":117630080}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6813","nonce":117630080}]},"public_addr":"172.21.15.114:6809/117630080","cluster_addr":"172.21.15.114:6811/117630080","heartbeat_back_addr":"172.21.15.114:6815/117630080","heartbeat_front_addr":"172.21.15.114:6813/117630080","state":["exists","up"]},{"osd":6,"uuid":"487cdd03-1123-4386-9237-97b2bc6dd92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6817","nonce":2139728372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6819","nonce":2139728372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6823","nonce":2139728372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6821","nonce":2139728372}]},"public_addr":"172.21.15.114:6817/2139728372","cluster_addr":"172.21.15.114:6819/2139728372","heartbeat_back_addr":"172.21.15.114:6823/2139728372","heartbeat_front_addr":"172.21.15.114:6821/2139728372","state":["exists","up"]},{"osd":7,"uuid":"0d46acbb-c271-4f1f-b5ba-090a44443308","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6825","nonce":3563377268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6827","nonce":3563377268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6831","nonce":3563377268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6829","nonce":3563377268}]},"public_addr":"172.21.15.114:6825/3563377268","cluster_addr":"172.21.15.114:6827/3563377268","heartbeat_back_addr":"172.21.15.114:6831/3563377268","heartbeat_front_addr":"172.21.15.114:6829/3563377268","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:38:46.298090+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:11.083177+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:35.186704+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:59.618196+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:21.910327+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:44.541214+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:08.064023+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:31.949563+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/3166816614":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/1996176058":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6801/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6801/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3631859869":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6800/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3110959800":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/806687008":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6800/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/721288352":"2024-04-13T08:37:02.805251+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T08:41:36.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:36 smithi090 ceph-mon[29100]: osdmap e57: 8 total, 8 up, 8 in 2024-04-12T08:41:36.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:36 smithi090 ceph-mon[29100]: osdmap e58: 8 total, 8 up, 8 in 2024-04-12T08:41:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:36 smithi090 ceph-mon[40003]: osdmap e57: 8 total, 8 up, 8 in 2024-04-12T08:41:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:36 smithi090 ceph-mon[40003]: osdmap e58: 8 total, 8 up, 8 in 2024-04-12T08:41:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:36 smithi114 ceph-mon[33573]: osdmap e57: 8 total, 8 up, 8 in 2024-04-12T08:41:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:36 smithi114 ceph-mon[33573]: osdmap e58: 8 total, 8 up, 8 in 2024-04-12T08:41:37.464 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-12T08:39:38.877058+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-12T08:41:37.464 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-12T08:41:37.715 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:41:37.837 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:37 smithi090 ceph-mon[40003]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:37.837 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:37 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1743179858' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:41:37.837 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:37 smithi090 ceph-mon[40003]: osdmap e59: 8 total, 8 up, 8 in 2024-04-12T08:41:37.838 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:37 smithi090 ceph-mon[29100]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:37.838 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:37 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1743179858' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:41:37.838 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:37 smithi090 ceph-mon[29100]: osdmap e59: 8 total, 8 up, 8 in 2024-04-12T08:41:38.018 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:37 smithi114 ceph-mon[33573]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:38.019 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:37 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1743179858' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:41:38.019 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:37 smithi114 ceph-mon[33573]: osdmap e59: 8 total, 8 up, 8 in 2024-04-12T08:41:39.649 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-12T08:41:39.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:39 smithi090 ceph-mon[29100]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:39.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:39 smithi090 ceph-mon[40003]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:40.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:39 smithi114 ceph-mon[33573]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:40.320 INFO:tasks.cephadm:Adding prometheus.a on smithi114 2024-04-12T08:41:40.320 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch apply prometheus '1;smithi114=a' 2024-04-12T08:41:40.581 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:41:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:40 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1116361437' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-12T08:41:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:40 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1116361437' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-12T08:41:40.965 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:40 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1116361437' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-12T08:41:41.829 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:41 smithi114 ceph-mon[33573]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T08:41:41.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:41 smithi090 ceph-mon[40003]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T08:41:41.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:41 smithi090 ceph-mon[29100]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T08:41:42.247 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled prometheus update... 2024-04-12T08:41:42.827 DEBUG:teuthology.orchestra.run.smithi114:prometheus.a> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@prometheus.a.service 2024-04-12T08:41:42.829 INFO:tasks.cephadm:Adding node-exporter.a on smithi090 2024-04-12T08:41:42.830 INFO:tasks.cephadm:Adding node-exporter.b on smithi114 2024-04-12T08:41:42.830 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch apply node-exporter '2;smithi090=a;smithi114=b' 2024-04-12T08:41:43.120 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:41:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:43 smithi114 ceph-mon[33573]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi114=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:43 smithi114 ceph-mon[33573]: Saving service prometheus spec with placement smithi114=a;count:1 2024-04-12T08:41:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:43.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:43.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:43.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:43 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:43.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:43 smithi114 ceph-mon[33573]: Deploying daemon prometheus.a on smithi114 2024-04-12T08:41:43.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:43 smithi114 ceph-mon[33573]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-12T08:41:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[29100]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi114=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[29100]: Saving service prometheus spec with placement smithi114=a;count:1 2024-04-12T08:41:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[29100]: Deploying daemon prometheus.a on smithi114 2024-04-12T08:41:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[29100]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-12T08:41:43.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[40003]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi114=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:43.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[40003]: Saving service prometheus spec with placement smithi114=a;count:1 2024-04-12T08:41:43.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:43.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:41:43.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:41:43.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:41:43.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:43.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[40003]: Deploying daemon prometheus.a on smithi114 2024-04-12T08:41:43.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:43 smithi090 ceph-mon[40003]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-12T08:41:45.564 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled node-exporter update... 2024-04-12T08:41:45.727 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:45 smithi090 ceph-mon[40003]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-12T08:41:45.727 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:45 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:45.728 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:45 smithi090 ceph-mon[29100]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-12T08:41:45.728 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:45 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:45 smithi114 ceph-mon[33573]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-12T08:41:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:45 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:46.670 DEBUG:teuthology.orchestra.run.smithi090:node-exporter.a> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.a.service 2024-04-12T08:41:46.673 DEBUG:teuthology.orchestra.run.smithi114:node-exporter.b> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.b.service 2024-04-12T08:41:46.675 INFO:tasks.cephadm:Adding alertmanager.a on smithi090 2024-04-12T08:41:46.675 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch apply alertmanager '1;smithi090=a' 2024-04-12T08:41:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:46 smithi090 ceph-mon[40003]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi090=a;smithi114=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:46 smithi090 ceph-mon[40003]: Saving service node-exporter spec with placement smithi090=a;smithi114=b;count:2 2024-04-12T08:41:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:46 smithi090 ceph-mon[29100]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi090=a;smithi114=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:46 smithi090 ceph-mon[29100]: Saving service node-exporter spec with placement smithi090=a;smithi114=b;count:2 2024-04-12T08:41:46.960 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:41:47.082 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:46 smithi114 ceph-mon[33573]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi090=a;smithi114=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:47.082 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:46 smithi114 ceph-mon[33573]: Saving service node-exporter spec with placement smithi090=a;smithi114=b;count:2 2024-04-12T08:41:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:47 smithi114 ceph-mon[33573]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T08:41:48.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:47 smithi090 ceph-mon[29100]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T08:41:48.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:47 smithi090 ceph-mon[40003]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T08:41:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:49 smithi114 ceph-mon[33573]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s, 0 objects/s recovering 2024-04-12T08:41:50.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:49 smithi090 ceph-mon[29100]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s, 0 objects/s recovering 2024-04-12T08:41:50.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:49 smithi090 ceph-mon[40003]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s, 0 objects/s recovering 2024-04-12T08:41:51.015 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled alertmanager update... 2024-04-12T08:41:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:52 smithi114 ceph-mon[33573]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-12T08:41:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:52 smithi114 ceph-mon[33573]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi090=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:52 smithi114 ceph-mon[33573]: Saving service alertmanager spec with placement smithi090=a;count:1 2024-04-12T08:41:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:52 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:52.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:52 smithi090 ceph-mon[40003]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-12T08:41:52.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:52 smithi090 ceph-mon[40003]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi090=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:52.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:52 smithi090 ceph-mon[40003]: Saving service alertmanager spec with placement smithi090=a;count:1 2024-04-12T08:41:52.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:52 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:52.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:51 smithi090 ceph-mon[29100]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-12T08:41:52.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:51 smithi090 ceph-mon[29100]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi090=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:41:52.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:51 smithi090 ceph-mon[29100]: Saving service alertmanager spec with placement smithi090=a;count:1 2024-04-12T08:41:52.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:51 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:54 smithi114 ceph-mon[33573]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:54.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:54 smithi090 ceph-mon[40003]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:54.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:54 smithi090 ceph-mon[29100]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:54.430 DEBUG:teuthology.orchestra.run.smithi090:alertmanager.a> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@alertmanager.a.service 2024-04-12T08:41:54.433 INFO:tasks.cephadm:Adding grafana.a on smithi114 2024-04-12T08:41:54.433 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph orch apply grafana '1;smithi114=a' 2024-04-12T08:41:54.979 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:41:55.102 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:41:54 smithi114 systemd[1]: Started Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:41:56.262 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:56 smithi114 ceph-mon[33573]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:56.263 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:56 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:56.263 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:56 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:56.263 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:56 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:56.263 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:56 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-12T08:41:56.263 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:56 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ignoring --setuser ceph since I am not root 2024-04-12T08:41:56.263 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:56 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ignoring --setgroup ceph since I am not root 2024-04-12T08:41:56.263 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:56 smithi114 ceph-mgr[35312]: -- 172.21.15.114:0/1956904194 <== mon.1 v2:172.21.15.114:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562807f289c0 con 0x562807efec00 2024-04-12T08:41:56.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mon[40003]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:56.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:56.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:56.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:56.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-12T08:41:56.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ignoring --setuser ceph since I am not root 2024-04-12T08:41:56.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ignoring --setgroup ceph since I am not root 2024-04-12T08:41:56.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mgr[29337]: -- 172.21.15.90:0/2017524469 <== mon.2 v2:172.21.15.90:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559a9a6509c0 con 0x559a9a626800 2024-04-12T08:41:56.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mon[29100]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:41:56.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:56.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:56.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' 2024-04-12T08:41:56.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-12T08:41:56.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:56 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:56.614+0000 7fb9452121c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:41:56.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:56 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:56.707+0000 7fb9452121c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:41:56.922 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:56.605+0000 7ff3a38b71c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:41:56.922 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:56.696+0000 7ff3a38b71c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:41:57.321 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:57 smithi090 ceph-mon[29100]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-12T08:41:57.321 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:41:57 smithi090 ceph-mon[29100]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-12T08:41:57.321 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:57 smithi090 ceph-mon[40003]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-12T08:41:57.321 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:41:57 smithi090 ceph-mon[40003]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-12T08:41:57.337 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:57 smithi114 ceph-mon[33573]: from='mgr.14152 172.21.15.90:0/4154675068' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-12T08:41:57.337 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:41:57 smithi114 ceph-mon[33573]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-12T08:41:57.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:57 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:57.335+0000 7fb9452121c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:41:57.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:57 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:57.425+0000 7fb9452121c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:41:57.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:57 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:57.508+0000 7fb9452121c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:41:57.642 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:57.320+0000 7ff3a38b71c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:41:57.642 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:57.408+0000 7ff3a38b71c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:41:57.642 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:57.490+0000 7ff3a38b71c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:41:57.642 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:57.641+0000 7ff3a38b71c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:41:57.922 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:57.720+0000 7ff3a38b71c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:41:57.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:57.889+0000 7ff3a38b71c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:41:58.088 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:57 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:57.659+0000 7fb9452121c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:41:58.088 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:57 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:57.740+0000 7fb9452121c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:41:58.089 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:57 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:57.909+0000 7fb9452121c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:41:58.089 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:58.087+0000 7fb9452121c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:41:58.328 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:58.068+0000 7ff3a38b71c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:41:58.328 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:58.151+0000 7ff3a38b71c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:41:58.328 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:58.241+0000 7ff3a38b71c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:41:58.345 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:58.168+0000 7fb9452121c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:41:58.345 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:58.259+0000 7fb9452121c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:41:58.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:58.344+0000 7fb9452121c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:41:58.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:58.327+0000 7ff3a38b71c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:41:59.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:59 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:59.362+0000 7ff3a38b71c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:41:59.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:59 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:59.490+0000 7ff3a38b71c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:41:59.693 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:59 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:59.372+0000 7fb9452121c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:41:59.693 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:59 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:59.500+0000 7fb9452121c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:41:59.694 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:59 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:59.692+0000 7fb9452121c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:42:00.085 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:59 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:59.682+0000 7ff3a38b71c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:42:00.085 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:41:59 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:41:59.908+0000 7ff3a38b71c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:42:00.091 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:41:59 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:41:59.916+0000 7fb9452121c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:42:00.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:00 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:42:00.090+0000 7fb9452121c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:42:00.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:00 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:42:00.269+0000 7fb9452121c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:42:00.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:42:00.084+0000 7ff3a38b71c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:42:00.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:42:00.266+0000 7ff3a38b71c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:42:00.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:00 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:42:00.529+0000 7fb9452121c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:42:00.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:00 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:42:00.613+0000 7fb9452121c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:42:00.922 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:42:00.529+0000 7ff3a38b71c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:42:00.922 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:42:00.615+0000 7ff3a38b71c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:42:01.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:00 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:42:00.996+0000 7fb9452121c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:42:01.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:01 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:42:01.001+0000 7ff3a38b71c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:42:02.061 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:01 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:42:01.769+0000 7fb9452121c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:42:02.079 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:01 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:42:01.783+0000 7ff3a38b71c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:42:02.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:42:02.059+0000 7fb9452121c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:42:02.391 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:42:02.079+0000 7ff3a38b71c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:42:02.650 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:42:02.390+0000 7ff3a38b71c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:42:02.650 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:42:02.587+0000 7ff3a38b71c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:42:02.650 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: Standby manager daemon x restarted 2024-04-12T08:42:02.650 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: Standby manager daemon x started 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: Active manager daemon y restarted 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: Activating manager daemon y 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: osdmap e60: 8 total, 8 up, 8 in 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: mgrmap e17: y(active, starting, since 0.00702823s), standbys: x 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:42:02.651 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[40003]: Manager daemon y is now available 2024-04-12T08:42:02.652 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: Standby manager daemon x restarted 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: Standby manager daemon x started 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: Active manager daemon y restarted 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: Activating manager daemon y 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: osdmap e60: 8 total, 8 up, 8 in 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: mgrmap e17: y(active, starting, since 0.00702823s), standbys: x 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:42:02.653 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:42:02.654 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-mon[29100]: Manager daemon y is now available 2024-04-12T08:42:02.672 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:42:02.370+0000 7fb9452121c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:42:02.673 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:42:02.564+0000 7fb9452121c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:42:02.673 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: [12/Apr/2024:08:42:02] ENGINE Bus STARTING 2024-04-12T08:42:02.673 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: Standby manager daemon x restarted 2024-04-12T08:42:02.673 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: Standby manager daemon x started 2024-04-12T08:42:02.673 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:42:02.673 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:42:02.673 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:42:02.673 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/1504362265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:42:02.673 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: Active manager daemon y restarted 2024-04-12T08:42:02.673 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: Activating manager daemon y 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: osdmap e60: 8 total, 8 up, 8 in 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: mgrmap e17: y(active, starting, since 0.00702823s), standbys: x 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:42:02.674 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:42:02.675 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:42:02.675 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:42:02.675 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:42:02.675 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:42:02.675 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:42:02.675 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-mon[33573]: Manager daemon y is now available 2024-04-12T08:42:02.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:42:02] ENGINE Bus STARTING 2024-04-12T08:42:02.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: CherryPy Checker: 2024-04-12T08:42:02.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: The Application mounted at '' has an empty config. 2024-04-12T08:42:02.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:42:02.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:42:02] ENGINE Serving on http://:::9283 2024-04-12T08:42:02.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:02 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:42:02] ENGINE Bus STARTED 2024-04-12T08:42:03.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: CherryPy Checker: 2024-04-12T08:42:03.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: The Application mounted at '' has an empty config. 2024-04-12T08:42:03.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: 2024-04-12T08:42:03.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: [12/Apr/2024:08:42:02] ENGINE Serving on http://:::9283 2024-04-12T08:42:03.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: [12/Apr/2024:08:42:02] ENGINE Bus STARTED 2024-04-12T08:42:03.610 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled grafana update... 2024-04-12T08:42:03.802 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:42:03.802 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:42:03.803 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:03.803 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:03.803 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:42:03.803 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:42:03.803 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:03 smithi114 ceph-mon[33573]: mgrmap e18: y(active, since 1.01112s), standbys: x 2024-04-12T08:42:03.803 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:42:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:42:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:42:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:42:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[40003]: mgrmap e18: y(active, since 1.01112s), standbys: x 2024-04-12T08:42:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:42:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:42:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:42:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:42:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[29100]: mgrmap e18: y(active, since 1.01112s), standbys: x 2024-04-12T08:42:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:04.416 DEBUG:teuthology.orchestra.run.smithi114:grafana.a> sudo journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service 2024-04-12T08:42:04.418 INFO:tasks.cephadm:Setting up client nodes... 2024-04-12T08:42:04.418 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-12T08:42:04.682 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:05.061 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[40003]: Saving service grafana spec with placement smithi114=a;count:1 2024-04-12T08:42:05.061 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[40003]: [12/Apr/2024:08:42:03] ENGINE Bus STARTING 2024-04-12T08:42:05.061 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[40003]: [12/Apr/2024:08:42:03] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:42:05.061 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[40003]: [12/Apr/2024:08:42:03] ENGINE Bus STARTED 2024-04-12T08:42:05.061 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:05.061 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:05.061 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:05.062 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[29100]: Saving service grafana spec with placement smithi114=a;count:1 2024-04-12T08:42:05.062 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[29100]: [12/Apr/2024:08:42:03] ENGINE Bus STARTING 2024-04-12T08:42:05.062 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[29100]: [12/Apr/2024:08:42:03] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:42:05.062 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[29100]: [12/Apr/2024:08:42:03] ENGINE Bus STARTED 2024-04-12T08:42:05.062 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:05.062 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:05.062 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:04 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:05.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:05 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:42:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:42:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:04 smithi114 ceph-mon[33573]: Saving service grafana spec with placement smithi114=a;count:1 2024-04-12T08:42:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:04 smithi114 ceph-mon[33573]: [12/Apr/2024:08:42:03] ENGINE Bus STARTING 2024-04-12T08:42:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:04 smithi114 ceph-mon[33573]: [12/Apr/2024:08:42:03] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:42:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:04 smithi114 ceph-mon[33573]: [12/Apr/2024:08:42:03] ENGINE Bus STARTED 2024-04-12T08:42:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:04 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:04 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:04 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[40003]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[40003]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[29100]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[29100]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:05 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:05 smithi114 ceph-mon[33573]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:05 smithi114 ceph-mon[33573]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-12T08:42:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:05 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:05 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:05 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:05 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:06.750 INFO:teuthology.orchestra.run.smithi090.stdout:[client.0] 2024-04-12T08:42:06.751 INFO:teuthology.orchestra.run.smithi090.stdout: key = AQDe8xhmSEKFLBAABsacM6ACM0FnfduxmDlEIQ== 2024-04-12T08:42:07.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi114 to 4016M 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2032191642' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2032191642' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi114 to 4016M 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:42:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2032191642' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2032191642' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi114 to 4016M 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2032191642' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2032191642' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:07.559 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-12T08:42:07.559 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-12T08:42:07.559 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-12T08:42:07.603 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-12T08:42:07.854 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[40003]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[40003]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[40003]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[40003]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[40003]: Deploying daemon alertmanager.a on smithi090 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[40003]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[29100]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:42:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[29100]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:42:08.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[29100]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:42:08.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[29100]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:42:08.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[29100]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:08.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[29100]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:42:08.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[29100]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:42:08.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[29100]: Deploying daemon alertmanager.a on smithi090 2024-04-12T08:42:08.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:07 smithi090 ceph-mon[29100]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-12T08:42:08.293 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:07 smithi114 ceph-mon[33573]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:42:08.293 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:07 smithi114 ceph-mon[33573]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:42:08.294 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:07 smithi114 ceph-mon[33573]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:42:08.294 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:07 smithi114 ceph-mon[33573]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:42:08.294 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:07 smithi114 ceph-mon[33573]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:08.294 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:07 smithi114 ceph-mon[33573]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:42:08.294 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:07 smithi114 ceph-mon[33573]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:42:08.294 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:07 smithi114 ceph-mon[33573]: Deploying daemon alertmanager.a on smithi090 2024-04-12T08:42:08.294 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:07 smithi114 ceph-mon[33573]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-12T08:42:09.701 INFO:teuthology.orchestra.run.smithi114.stdout:[client.1] 2024-04-12T08:42:09.701 INFO:teuthology.orchestra.run.smithi114.stdout: key = AQDh8xhmwa6aKRAA6taVQpt8EPBOpx0pg1NWnw== 2024-04-12T08:42:10.329 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:09 smithi114 ceph-mon[33573]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:10.329 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:09 smithi114 ceph-mon[33573]: from='client.? 172.21.15.114:0/3591539014' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T08:42:10.329 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:09 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T08:42:10.329 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:09 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T08:42:10.352 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-12T08:42:10.352 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-12T08:42:10.353 DEBUG:teuthology.orchestra.run.smithi114:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-12T08:42:10.394 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-12T08:42:10.395 DEBUG:teuthology.orchestra.run.smithi090:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-12T08:42:10.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:09 smithi090 ceph-mon[40003]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:10.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:09 smithi090 ceph-mon[40003]: from='client.? 172.21.15.114:0/3591539014' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T08:42:10.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:09 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T08:42:10.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:09 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T08:42:10.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:09 smithi090 ceph-mon[29100]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:10.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:09 smithi090 ceph-mon[29100]: from='client.? 172.21.15.114:0/3591539014' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T08:42:10.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:09 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T08:42:10.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:09 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T08:42:10.686 INFO:teuthology.misc.health.smithi090.stdout: 2024-04-12T08:42:10.686 INFO:teuthology.misc.health.smithi090.stdout:{"epoch":60,"fsid":"96735376-f8a7-11ee-bc8c-c7b262605968","created":"2024-04-12T08:36:35.370883+0000","modified":"2024-04-12T08:42:02.589158+0000","last_up_change":"2024-04-12T08:41:33.609734+0000","last_in_change":"2024-04-12T08:41:16.832300+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T08:39:38.877058+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"3e888cb8-b0cb-4cb8-95ae-7dd29d090968","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6803","nonce":937815701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6805","nonce":937815701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6809","nonce":937815701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6807","nonce":937815701}]},"public_addr":"172.21.15.90:6803/937815701","cluster_addr":"172.21.15.90:6805/937815701","heartbeat_back_addr":"172.21.15.90:6809/937815701","heartbeat_front_addr":"172.21.15.90:6807/937815701","state":["exists","up"]},{"osd":1,"uuid":"637213e4-7019-4430-ab01-b7196988f82c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6811","nonce":2625409332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6813","nonce":2625409332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6817","nonce":2625409332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6815","nonce":2625409332}]},"public_addr":"172.21.15.90:6811/2625409332","cluster_addr":"172.21.15.90:6813/2625409332","heartbeat_back_addr":"172.21.15.90:6817/2625409332","heartbeat_front_addr":"172.21.15.90:6815/2625409332","state":["exists","up"]},{"osd":2,"uuid":"99513c85-6468-4377-85b8-3dc52f0d0f2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6819","nonce":3723833287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6821","nonce":3723833287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6825","nonce":3723833287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6823","nonce":3723833287}]},"public_addr":"172.21.15.90:6819/3723833287","cluster_addr":"172.21.15.90:6821/3723833287","heartbeat_back_addr":"172.21.15.90:6825/3723833287","heartbeat_front_addr":"172.21.15.90:6823/3723833287","state":["exists","up"]},{"osd":3,"uuid":"afd01784-8987-4954-9be5-57d950beb611","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6827","nonce":1425929386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6829","nonce":1425929386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6833","nonce":1425929386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6831","nonce":1425929386}]},"public_addr":"172.21.15.90:6827/1425929386","cluster_addr":"172.21.15.90:6829/1425929386","heartbeat_back_addr":"172.21.15.90:6833/1425929386","heartbeat_front_addr":"172.21.15.90:6831/1425929386","state":["exists","up"]},{"osd":4,"uuid":"596f9e87-d99e-423c-902f-f14c5dcdcf36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6801","nonce":1802592095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6803","nonce":1802592095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6807","nonce":1802592095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6805","nonce":1802592095}]},"public_addr":"172.21.15.114:6801/1802592095","cluster_addr":"172.21.15.114:6803/1802592095","heartbeat_back_addr":"172.21.15.114:6807/1802592095","heartbeat_front_addr":"172.21.15.114:6805/1802592095","state":["exists","up"]},{"osd":5,"uuid":"b2374027-89a9-4670-9593-3ddf9393b286","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6809","nonce":117630080}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6811","nonce":117630080}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6815","nonce":117630080}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6813","nonce":117630080}]},"public_addr":"172.21.15.114:6809/117630080","cluster_addr":"172.21.15.114:6811/117630080","heartbeat_back_addr":"172.21.15.114:6815/117630080","heartbeat_front_addr":"172.21.15.114:6813/117630080","state":["exists","up"]},{"osd":6,"uuid":"487cdd03-1123-4386-9237-97b2bc6dd92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6817","nonce":2139728372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6819","nonce":2139728372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6823","nonce":2139728372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6821","nonce":2139728372}]},"public_addr":"172.21.15.114:6817/2139728372","cluster_addr":"172.21.15.114:6819/2139728372","heartbeat_back_addr":"172.21.15.114:6823/2139728372","heartbeat_front_addr":"172.21.15.114:6821/2139728372","state":["exists","up"]},{"osd":7,"uuid":"0d46acbb-c271-4f1f-b5ba-090a44443308","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6825","nonce":3563377268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6827","nonce":3563377268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6831","nonce":3563377268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6829","nonce":3563377268}]},"public_addr":"172.21.15.114:6825/3563377268","cluster_addr":"172.21.15.114:6827/3563377268","heartbeat_back_addr":"172.21.15.114:6831/3563377268","heartbeat_front_addr":"172.21.15.114:6829/3563377268","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:38:46.298090+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:11.083177+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:35.186704+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:59.618196+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:21.910327+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:44.541214+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:08.064023+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:31.949563+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/3964952044":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6800/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/3166816614":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/1996176058":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6801/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3357701831":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3631859869":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6800/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3110959800":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/806687008":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6800/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3408352967":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/577805764":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/721288352":"2024-04-13T08:37:02.805251+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T08:42:10.696 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-12T08:42:10.696 INFO:tasks.cephadm:Creating RBD pool 2024-04-12T08:42:10.697 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-12T08:42:11.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:10 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2809280512' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:42:11.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:10 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2809280512' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:42:11.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:10 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2809280512' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:42:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:11 smithi114 ceph-mon[33573]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-12T08:42:12.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:11 smithi090 ceph-mon[40003]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-12T08:42:12.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:11 smithi090 ceph-mon[29100]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-12T08:42:14.034 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:14.162 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:13 smithi090 ceph-mon[40003]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-12T08:42:14.162 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:13 smithi090 ceph-mon[29100]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-12T08:42:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:13 smithi114 ceph-mon[33573]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-12T08:42:15.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:42:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:42:15.923 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:15 smithi090 podman[68555]: 2024-04-12 08:42:15.83499 +0000 UTC m=+0.951252092 container create 3d4f8e008936ca7af0c21c8e7737572bfe02776150eed319f6c29c7ac03b03a2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:42:16.240 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:15 smithi090 ceph-mon[40003]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-12T08:42:16.240 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:15 smithi090 ceph-mon[29100]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-12T08:42:16.240 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 podman[68555]: 2024-04-12 08:42:16.14798508 +0000 UTC m=+1.264247169 container init 3d4f8e008936ca7af0c21c8e7737572bfe02776150eed319f6c29c7ac03b03a2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:42:16.240 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 podman[68555]: 2024-04-12 08:42:16.150654662 +0000 UTC m=+1.266916753 container start 3d4f8e008936ca7af0c21c8e7737572bfe02776150eed319f6c29c7ac03b03a2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:42:16.240 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:42:16.162Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-12T08:42:16.241 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:42:16.162Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-12T08:42:16.241 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:42:16.163Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.90 port=9094 2024-04-12T08:42:16.241 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:42:16.164Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-12T08:42:16.241 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:42:16.187Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-12T08:42:16.241 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:42:16.188Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-12T08:42:16.241 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:42:16.190Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-12T08:42:16.241 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:42:16.190Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-12T08:42:16.241 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 bash[68555]: 3d4f8e008936ca7af0c21c8e7737572bfe02776150eed319f6c29c7ac03b03a2 2024-04-12T08:42:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:15 smithi114 ceph-mon[33573]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-12T08:42:16.566 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:16 smithi090 systemd[1]: Started Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:42:17.173 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:17 smithi090 systemd[1]: Starting Ceph node-exporter.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:42:17.334 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'rbd' created 2024-04-12T08:42:17.571 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.571 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.571 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.571 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[29100]: Deploying daemon node-exporter.a on smithi090 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[29100]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3628408935' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[40003]: Deploying daemon node-exporter.a on smithi090 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[40003]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T08:42:17.572 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3628408935' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-12T08:42:17.573 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:17 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-12T08:42:17.573 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:17 smithi090 bash[68877]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-12T08:42:17.573 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:17 smithi090 bash[68877]: Getting image source signatures 2024-04-12T08:42:17.573 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:17 smithi090 bash[68877]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-12T08:42:17.573 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:17 smithi090 bash[68877]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-12T08:42:17.573 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:17 smithi090 bash[68877]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-12T08:42:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:17 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:17 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:17 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:17 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:17 smithi114 ceph-mon[33573]: Deploying daemon node-exporter.a on smithi090 2024-04-12T08:42:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:17 smithi114 ceph-mon[33573]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T08:42:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:17 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3628408935' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-12T08:42:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:17 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-12T08:42:18.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:18 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:42:18.165Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000573115s 2024-04-12T08:42:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:18 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-12T08:42:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:18 smithi090 ceph-mon[29100]: osdmap e61: 8 total, 8 up, 8 in 2024-04-12T08:42:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:18 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:18 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:18.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:18 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-12T08:42:18.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:18 smithi090 ceph-mon[40003]: osdmap e61: 8 total, 8 up, 8 in 2024-04-12T08:42:18.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:18 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:18.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:18 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:18.543 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-12T08:42:18.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:18 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-12T08:42:18.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:18 smithi114 ceph-mon[33573]: osdmap e61: 8 total, 8 up, 8 in 2024-04-12T08:42:18.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:18 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:18.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:18 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:19.124 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:19.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:19 smithi090 ceph-mon[40003]: osdmap e62: 8 total, 8 up, 8 in 2024-04-12T08:42:19.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:19 smithi090 ceph-mon[40003]: pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:19.423 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:19 smithi090 bash[68877]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-12T08:42:19.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:19 smithi090 ceph-mon[29100]: osdmap e62: 8 total, 8 up, 8 in 2024-04-12T08:42:19.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:19 smithi090 ceph-mon[29100]: pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:19.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:19 smithi114 ceph-mon[33573]: osdmap e62: 8 total, 8 up, 8 in 2024-04-12T08:42:19.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:19 smithi114 ceph-mon[33573]: pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:19.684 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:19 smithi090 bash[68877]: Writing manifest to image destination 2024-04-12T08:42:20.337 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 podman[68877]: 2024-04-12 08:42:19.983527263 +0000 UTC m=+2.845699444 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-12T08:42:20.337 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 podman[68877]: 2024-04-12 08:42:20.20934185 +0000 UTC m=+3.071514027 container create b71f7d0a7569f877254bcb7090537fe2aeaabbbc491e6c5b24dfd615b7b7629e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T08:42:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:20 smithi114 ceph-mon[33573]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:42:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:20 smithi114 ceph-mon[33573]: osdmap e63: 8 total, 8 up, 8 in 2024-04-12T08:42:20.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-mon[29100]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:42:20.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-mon[29100]: osdmap e63: 8 total, 8 up, 8 in 2024-04-12T08:42:20.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-mon[40003]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:42:20.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-mon[40003]: osdmap e63: 8 total, 8 up, 8 in 2024-04-12T08:42:20.673 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 podman[68877]: 2024-04-12 08:42:20.453953907 +0000 UTC m=+3.316126092 container init b71f7d0a7569f877254bcb7090537fe2aeaabbbc491e6c5b24dfd615b7b7629e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T08:42:20.673 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 podman[68877]: 2024-04-12 08:42:20.457133058 +0000 UTC m=+3.319305235 container start b71f7d0a7569f877254bcb7090537fe2aeaabbbc491e6c5b24dfd615b7b7629e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T08:42:20.673 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.461Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.461Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.462Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.462Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.462Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.462Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=arp 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-12T08:42:20.674 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-12T08:42:20.675 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-12T08:42:20.675 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-12T08:42:20.675 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-12T08:42:20.675 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=edac 2024-04-12T08:42:20.675 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-12T08:42:20.675 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-12T08:42:20.675 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-12T08:42:20.676 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-12T08:42:20.676 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-12T08:42:20.676 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-12T08:42:20.676 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-12T08:42:20.676 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-12T08:42:20.676 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-12T08:42:20.676 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-12T08:42:20.676 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-12T08:42:20.676 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-12T08:42:20.676 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=os 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=stat 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-12T08:42:20.677 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=time 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=uname 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[69033]: ts=2024-04-12T08:42:20.463Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 bash[68877]: b71f7d0a7569f877254bcb7090537fe2aeaabbbc491e6c5b24dfd615b7b7629e 2024-04-12T08:42:20.678 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:42:20 smithi090 systemd[1]: Started Ceph node-exporter.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:42:21.602 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:21 smithi114 bash[62417]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-12T08:42:21.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:21 smithi114 ceph-mon[33573]: pgmap v15: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:21.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:21.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:21.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:21.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:21 smithi114 ceph-mon[33573]: Deploying daemon node-exporter.b on smithi114 2024-04-12T08:42:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:21 smithi090 ceph-mon[40003]: pgmap v15: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:21 smithi090 ceph-mon[40003]: Deploying daemon node-exporter.b on smithi114 2024-04-12T08:42:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:21 smithi090 ceph-mon[29100]: pgmap v15: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:21.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:21.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:21.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:21 smithi090 ceph-mon[29100]: Deploying daemon node-exporter.b on smithi114 2024-04-12T08:42:22.367 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:21 smithi114 bash[62417]: Getting image source signatures 2024-04-12T08:42:22.367 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:21 smithi114 bash[62417]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-12T08:42:22.367 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:21 smithi114 bash[62417]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-12T08:42:22.367 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:21 smithi114 bash[62417]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-12T08:42:22.644 INFO:teuthology.orchestra.run.smithi090.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-12T08:42:22.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:22 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/415514779' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:42:22.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:22 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:42:22.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:22 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/415514779' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:42:22.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:22 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:42:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:22 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/415514779' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:42:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:22 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:42:23.626 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-12T08:42:23.627 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-12T08:42:23.627 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph mgr dump --format=json 2024-04-12T08:42:23.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:23 smithi114 ceph-mon[33573]: pgmap v16: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:23.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:23 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:42:23.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:23 smithi114 ceph-mon[33573]: osdmap e64: 8 total, 8 up, 8 in 2024-04-12T08:42:23.867 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:23 smithi114 bash[62417]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-12T08:42:23.867 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:23 smithi114 bash[62417]: Writing manifest to image destination 2024-04-12T08:42:23.884 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:23 smithi090 ceph-mon[40003]: pgmap v16: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:23 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:42:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:23 smithi090 ceph-mon[40003]: osdmap e64: 8 total, 8 up, 8 in 2024-04-12T08:42:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:23 smithi090 ceph-mon[29100]: pgmap v16: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:23 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:42:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:23 smithi090 ceph-mon[29100]: osdmap e64: 8 total, 8 up, 8 in 2024-04-12T08:42:24.283 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:23 smithi114 podman[62417]: 2024-04-12 08:42:23.899269115 +0000 UTC m=+2.460132053 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-12T08:42:24.283 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 podman[62417]: 2024-04-12 08:42:24.005676538 +0000 UTC m=+2.566539463 container create 9a6215e32fcfa99b2b91e57343f9dcbd81202a8e95dd84211aa6a9e658ac457c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T08:42:24.283 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 podman[62417]: 2024-04-12 08:42:24.199511273 +0000 UTC m=+2.760374199 container init 9a6215e32fcfa99b2b91e57343f9dcbd81202a8e95dd84211aa6a9e658ac457c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T08:42:24.283 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 podman[62417]: 2024-04-12 08:42:24.202186722 +0000 UTC m=+2.763049656 container start 9a6215e32fcfa99b2b91e57343f9dcbd81202a8e95dd84211aa6a9e658ac457c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.206Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.206Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.206Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.206Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=arp 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-12T08:42:24.284 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-12T08:42:24.285 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-12T08:42:24.285 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-12T08:42:24.285 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-12T08:42:24.285 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-12T08:42:24.285 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-12T08:42:24.285 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=edac 2024-04-12T08:42:24.285 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-12T08:42:24.285 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-12T08:42:24.285 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-12T08:42:24.285 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=os 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-12T08:42:24.286 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=stat 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=time 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=uname 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.207Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-12T08:42:24.287 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.208Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-12T08:42:24.288 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[62466]: ts=2024-04-12T08:42:24.208Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-12T08:42:24.288 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 bash[62417]: 9a6215e32fcfa99b2b91e57343f9dcbd81202a8e95dd84211aa6a9e658ac457c 2024-04-12T08:42:24.617 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:42:24 smithi114 systemd[1]: Started Ceph node-exporter.b for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:42:24.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:42:24] ENGINE Bus STOPPING 2024-04-12T08:42:24.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:42:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T08:42:24.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:42:24] ENGINE Bus STOPPED 2024-04-12T08:42:24.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:42:24] ENGINE Bus STARTING 2024-04-12T08:42:24.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:42:24] ENGINE Serving on http://:::9283 2024-04-12T08:42:24.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:42:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:42:24] ENGINE Bus STARTED 2024-04-12T08:42:25.347 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:42:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:42:25.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T08:42:25.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T08:42:25.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: Deploying daemon grafana.a on smithi114 2024-04-12T08:42:25.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:25 smithi114 ceph-mon[33573]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:25.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: Deploying daemon grafana.a on smithi114 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[29100]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:25.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: Deploying daemon grafana.a on smithi114 2024-04-12T08:42:25.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:25 smithi090 ceph-mon[40003]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:25.807 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:42:26.348 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:42:26 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:42:26.168Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003242628s 2024-04-12T08:42:26.478 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":20,"active_gid":14421,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6800","nonce":3500986148},{"type":"v1","addr":"172.21.15.90:6801","nonce":3500986148}]},"active_addr":"172.21.15.90:6801/3500986148","active_change":"2024-04-12T08:42:02.589309+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24328,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.90:8443/","prometheus":"http://172.21.15.90:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":1763471199}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":794297450}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":3912205664}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":4275803179}]}]} 2024-04-12T08:42:26.482 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-12T08:42:26.482 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-12T08:42:26.482 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T08:42:26.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:26 smithi114 ceph-mon[33573]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:42:26.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:26 smithi114 ceph-mon[33573]: Cluster is now healthy 2024-04-12T08:42:26.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:26 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3822607707' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T08:42:26.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:26 smithi090 ceph-mon[40003]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:42:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:26 smithi090 ceph-mon[40003]: Cluster is now healthy 2024-04-12T08:42:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:26 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3822607707' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T08:42:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:26 smithi090 ceph-mon[29100]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:42:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:26 smithi090 ceph-mon[29100]: Cluster is now healthy 2024-04-12T08:42:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:26 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3822607707' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T08:42:26.737 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:27.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:27 smithi090 ceph-mon[29100]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:27.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:27 smithi090 ceph-mon[40003]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:27.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:27 smithi114 ceph-mon[33573]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:28.669 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:42:28.669 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":64,"fsid":"96735376-f8a7-11ee-bc8c-c7b262605968","created":"2024-04-12T08:36:35.370883+0000","modified":"2024-04-12T08:42:22.629102+0000","last_up_change":"2024-04-12T08:41:33.609734+0000","last_in_change":"2024-04-12T08:41:16.832300+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T08:39:38.877058+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T08:42:17.012082+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"3e888cb8-b0cb-4cb8-95ae-7dd29d090968","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6803","nonce":937815701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6805","nonce":937815701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6809","nonce":937815701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6807","nonce":937815701}]},"public_addr":"172.21.15.90:6803/937815701","cluster_addr":"172.21.15.90:6805/937815701","heartbeat_back_addr":"172.21.15.90:6809/937815701","heartbeat_front_addr":"172.21.15.90:6807/937815701","state":["exists","up"]},{"osd":1,"uuid":"637213e4-7019-4430-ab01-b7196988f82c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6811","nonce":2625409332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6813","nonce":2625409332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6817","nonce":2625409332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6815","nonce":2625409332}]},"public_addr":"172.21.15.90:6811/2625409332","cluster_addr":"172.21.15.90:6813/2625409332","heartbeat_back_addr":"172.21.15.90:6817/2625409332","heartbeat_front_addr":"172.21.15.90:6815/2625409332","state":["exists","up"]},{"osd":2,"uuid":"99513c85-6468-4377-85b8-3dc52f0d0f2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6819","nonce":3723833287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6821","nonce":3723833287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6825","nonce":3723833287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6823","nonce":3723833287}]},"public_addr":"172.21.15.90:6819/3723833287","cluster_addr":"172.21.15.90:6821/3723833287","heartbeat_back_addr":"172.21.15.90:6825/3723833287","heartbeat_front_addr":"172.21.15.90:6823/3723833287","state":["exists","up"]},{"osd":3,"uuid":"afd01784-8987-4954-9be5-57d950beb611","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6827","nonce":1425929386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6829","nonce":1425929386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6833","nonce":1425929386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6831","nonce":1425929386}]},"public_addr":"172.21.15.90:6827/1425929386","cluster_addr":"172.21.15.90:6829/1425929386","heartbeat_back_addr":"172.21.15.90:6833/1425929386","heartbeat_front_addr":"172.21.15.90:6831/1425929386","state":["exists","up"]},{"osd":4,"uuid":"596f9e87-d99e-423c-902f-f14c5dcdcf36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6801","nonce":1802592095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6803","nonce":1802592095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6807","nonce":1802592095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6805","nonce":1802592095}]},"public_addr":"172.21.15.114:6801/1802592095","cluster_addr":"172.21.15.114:6803/1802592095","heartbeat_back_addr":"172.21.15.114:6807/1802592095","heartbeat_front_addr":"172.21.15.114:6805/1802592095","state":["exists","up"]},{"osd":5,"uuid":"b2374027-89a9-4670-9593-3ddf9393b286","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6809","nonce":117630080}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6811","nonce":117630080}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6815","nonce":117630080}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6813","nonce":117630080}]},"public_addr":"172.21.15.114:6809/117630080","cluster_addr":"172.21.15.114:6811/117630080","heartbeat_back_addr":"172.21.15.114:6815/117630080","heartbeat_front_addr":"172.21.15.114:6813/117630080","state":["exists","up"]},{"osd":6,"uuid":"487cdd03-1123-4386-9237-97b2bc6dd92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6817","nonce":2139728372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6819","nonce":2139728372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6823","nonce":2139728372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6821","nonce":2139728372}]},"public_addr":"172.21.15.114:6817/2139728372","cluster_addr":"172.21.15.114:6819/2139728372","heartbeat_back_addr":"172.21.15.114:6823/2139728372","heartbeat_front_addr":"172.21.15.114:6821/2139728372","state":["exists","up"]},{"osd":7,"uuid":"0d46acbb-c271-4f1f-b5ba-090a44443308","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6825","nonce":3563377268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6827","nonce":3563377268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6831","nonce":3563377268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6829","nonce":3563377268}]},"public_addr":"172.21.15.114:6825/3563377268","cluster_addr":"172.21.15.114:6827/3563377268","heartbeat_back_addr":"172.21.15.114:6831/3563377268","heartbeat_front_addr":"172.21.15.114:6829/3563377268","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:38:46.298090+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:11.083177+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:35.186704+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:59.618196+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:21.910327+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:44.541214+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:08.064023+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:31.949563+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/3964952044":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6800/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/3166816614":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/1996176058":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6801/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3357701831":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3631859869":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6800/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3110959800":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/806687008":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6800/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3408352967":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/577805764":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/721288352":"2024-04-13T08:37:02.805251+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T08:42:28.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:28 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:28.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:28 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:29.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:28 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:29.699 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-12T08:42:29.700 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T08:42:29.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:29 smithi090 ceph-mon[40003]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:29.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:29 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1752527406' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:42:29.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:29 smithi090 ceph-mon[29100]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:29.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:29 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1752527406' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:42:29.958 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:29 smithi114 ceph-mon[33573]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:29 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1752527406' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:42:31.872 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:42:31.872 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":64,"fsid":"96735376-f8a7-11ee-bc8c-c7b262605968","created":"2024-04-12T08:36:35.370883+0000","modified":"2024-04-12T08:42:22.629102+0000","last_up_change":"2024-04-12T08:41:33.609734+0000","last_in_change":"2024-04-12T08:41:16.832300+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T08:39:38.877058+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T08:42:17.012082+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"3e888cb8-b0cb-4cb8-95ae-7dd29d090968","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6803","nonce":937815701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6805","nonce":937815701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6809","nonce":937815701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6807","nonce":937815701}]},"public_addr":"172.21.15.90:6803/937815701","cluster_addr":"172.21.15.90:6805/937815701","heartbeat_back_addr":"172.21.15.90:6809/937815701","heartbeat_front_addr":"172.21.15.90:6807/937815701","state":["exists","up"]},{"osd":1,"uuid":"637213e4-7019-4430-ab01-b7196988f82c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6811","nonce":2625409332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6813","nonce":2625409332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6817","nonce":2625409332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6815","nonce":2625409332}]},"public_addr":"172.21.15.90:6811/2625409332","cluster_addr":"172.21.15.90:6813/2625409332","heartbeat_back_addr":"172.21.15.90:6817/2625409332","heartbeat_front_addr":"172.21.15.90:6815/2625409332","state":["exists","up"]},{"osd":2,"uuid":"99513c85-6468-4377-85b8-3dc52f0d0f2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6819","nonce":3723833287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6821","nonce":3723833287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6825","nonce":3723833287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6823","nonce":3723833287}]},"public_addr":"172.21.15.90:6819/3723833287","cluster_addr":"172.21.15.90:6821/3723833287","heartbeat_back_addr":"172.21.15.90:6825/3723833287","heartbeat_front_addr":"172.21.15.90:6823/3723833287","state":["exists","up"]},{"osd":3,"uuid":"afd01784-8987-4954-9be5-57d950beb611","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6827","nonce":1425929386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6829","nonce":1425929386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6833","nonce":1425929386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6831","nonce":1425929386}]},"public_addr":"172.21.15.90:6827/1425929386","cluster_addr":"172.21.15.90:6829/1425929386","heartbeat_back_addr":"172.21.15.90:6833/1425929386","heartbeat_front_addr":"172.21.15.90:6831/1425929386","state":["exists","up"]},{"osd":4,"uuid":"596f9e87-d99e-423c-902f-f14c5dcdcf36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6801","nonce":1802592095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6803","nonce":1802592095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6807","nonce":1802592095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6805","nonce":1802592095}]},"public_addr":"172.21.15.114:6801/1802592095","cluster_addr":"172.21.15.114:6803/1802592095","heartbeat_back_addr":"172.21.15.114:6807/1802592095","heartbeat_front_addr":"172.21.15.114:6805/1802592095","state":["exists","up"]},{"osd":5,"uuid":"b2374027-89a9-4670-9593-3ddf9393b286","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6809","nonce":117630080}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6811","nonce":117630080}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6815","nonce":117630080}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6813","nonce":117630080}]},"public_addr":"172.21.15.114:6809/117630080","cluster_addr":"172.21.15.114:6811/117630080","heartbeat_back_addr":"172.21.15.114:6815/117630080","heartbeat_front_addr":"172.21.15.114:6813/117630080","state":["exists","up"]},{"osd":6,"uuid":"487cdd03-1123-4386-9237-97b2bc6dd92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6817","nonce":2139728372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6819","nonce":2139728372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6823","nonce":2139728372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6821","nonce":2139728372}]},"public_addr":"172.21.15.114:6817/2139728372","cluster_addr":"172.21.15.114:6819/2139728372","heartbeat_back_addr":"172.21.15.114:6823/2139728372","heartbeat_front_addr":"172.21.15.114:6821/2139728372","state":["exists","up"]},{"osd":7,"uuid":"0d46acbb-c271-4f1f-b5ba-090a44443308","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6825","nonce":3563377268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6827","nonce":3563377268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6831","nonce":3563377268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6829","nonce":3563377268}]},"public_addr":"172.21.15.114:6825/3563377268","cluster_addr":"172.21.15.114:6827/3563377268","heartbeat_back_addr":"172.21.15.114:6831/3563377268","heartbeat_front_addr":"172.21.15.114:6829/3563377268","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:38:46.298090+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:11.083177+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:35.186704+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:59.618196+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:21.910327+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:44.541214+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:08.064023+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:31.949563+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/3964952044":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6800/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/3166816614":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/1996176058":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6801/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3357701831":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3631859869":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6800/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3110959800":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/806687008":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6800/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3408352967":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/577805764":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/721288352":"2024-04-13T08:37:02.805251+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T08:42:31.872 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:31 smithi090 ceph-mon[40003]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:31.873 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:31 smithi090 ceph-mon[29100]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:32.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:31 smithi114 ceph-mon[33573]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:32.643 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-12T08:42:32.644 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-12T08:42:32.644 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-12T08:42:32.645 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-12T08:42:32.645 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-12T08:42:32.645 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-12T08:42:32.646 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-12T08:42:32.646 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-12T08:42:32.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:32 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3030166173' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:42:32.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:32 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:32.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:32 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3030166173' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:42:32.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:32 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:33.109 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:33.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:32 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3030166173' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:42:33.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:32 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:33.163 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:33.165 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:33.171 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:33.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:33 smithi090 ceph-mon[29100]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:33.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:33 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:33.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:33 smithi090 ceph-mon[40003]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:33.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:33 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:34.109 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:34.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:33 smithi114 ceph-mon[33573]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:34.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:33 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:34.124 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:34.124 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:34.125 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:35.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:42:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:42:35.784 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:35 smithi114 ceph-mon[33573]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:35.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:35 smithi090 ceph-mon[40003]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:35 smithi090 ceph-mon[29100]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:37.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:37 smithi114 ceph-mon[33573]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:37.867 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:37 smithi114 podman[62795]: 2024-04-12 08:42:37.713999554 +0000 UTC m=+0.016734851 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T08:42:38.095 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:37 smithi090 ceph-mon[40003]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:38.096 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:37 smithi090 ceph-mon[29100]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:38.157 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 podman[62795]: 2024-04-12 08:42:38.073889828 +0000 UTC m=+0.376625112 container init 44058eaefb748e9b75b348f9d15698ae73638d871d03120925904a0e4d27ec6c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, maintainer=Paul Cuzner , io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, release=239.1651231664, version=8.5, com.redhat.component=ubi8-container) 2024-04-12T08:42:38.157 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 podman[62795]: 2024-04-12 08:42:38.076510631 +0000 UTC m=+0.379245930 container start 44058eaefb748e9b75b348f9d15698ae73638d871d03120925904a0e4d27ec6c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, maintainer=Paul Cuzner , version=8.5, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, vcs-type=git, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container) 2024-04-12T08:42:38.409 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 bash[62795]: 44058eaefb748e9b75b348f9d15698ae73638d871d03120925904a0e4d27ec6c 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 systemd[1]: Started Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229305401Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229494054Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229513156Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229520944Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229527098Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229533163Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229539081Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229544787Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229596753Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229602991Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-12T08:42:38.410 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229608481Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.22961448Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.22962074Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=settings t=2024-04-12T08:42:38.229626639Z level=info msg="App mode production" 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=sqlstore t=2024-04-12T08:42:38.229671179Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=sqlstore t=2024-04-12T08:42:38.229686774Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.230506104Z level=info msg="Starting DB migrations" 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.230591533Z level=info msg="Executing migration" id="create migration_log table" 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.231175325Z level=info msg="Executing migration" id="create user table" 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.231639775Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.23209299Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.232530324Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.233141589Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.233491076Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.23418608Z level=info msg="Executing migration" id="create user table v2" 2024-04-12T08:42:38.411 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.234565719Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.234963121Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.235368534Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.235715763Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.23606036Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.236436318Z level=info msg="Executing migration" id="Update user table charset" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.236734365Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.237134117Z level=info msg="Executing migration" id="Add missing user data" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.237449054Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.237863775Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.238254384Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-12T08:42:38.412 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.238632642Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.239729785Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.240160788Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.24062895Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.241072199Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.241470081Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.241878261Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.242182862Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.242559151Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.242978755Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.243366732Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.243765974Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.244383622Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-12T08:42:38.414 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.244791689Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.245191787Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.245569002Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.245966919Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.246360216Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.24670291Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.247048468Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.247372197Z level=info msg="Executing migration" id="create star table" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.247762506Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.24819054Z level=info msg="Executing migration" id="create org table v1" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.248601842Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.249023482Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.24942178Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.249839503Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.250307082Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-12T08:42:38.415 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.250742886Z level=info msg="Executing migration" id="Update org table charset" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.251047247Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.251310351Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.251640293Z level=info msg="Executing migration" id="create dashboard table" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.252037051Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.252423832Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.252870358Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.253259331Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.255775105Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.256240674Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.257116765Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.257523344Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.257978772Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.258402583Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-12T08:42:38.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.258796794Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.259217326Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.259501656Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.260030813Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.26051251Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.261058744Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.261450361Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.261963158Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.262420418Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.262865458Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.263182185Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.263451115Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.264017944Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.264563932Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-12T08:42:38.417 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.267389957Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.270149208Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.27306797Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.275905692Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.278706346Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.281447902Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.284271101Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.287003024Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.28989503Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.292631059Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.295258077Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.297942826Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.300352897Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.303131903Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-12T08:42:38.418 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.305902684Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.308653071Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.311273917Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.314206073Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.316805244Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.319377915Z level=info msg="Executing migration" id="create data_source table" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.322357955Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.324964342Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.325411593Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.328251295Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.330814061Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.331857479Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.332661302Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-12T08:42:38.419 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.333529288Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.334353865Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.335183647Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.335989808Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.336858581Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.337705282Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.338539136Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.33941073Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.340246512Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.341111483Z level=info msg="Executing migration" id="Add uid column" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.341950925Z level=info msg="Executing migration" id="Update uid value" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.342838797Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.343715952Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.344456794Z level=info msg="Executing migration" id="create api_key table" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.345319381Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-12T08:42:38.420 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.346165725Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.347019034Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.347912698Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.348781468Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.349198143Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.349953224Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.350996317Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.351477043Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.352250565Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.352716136Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.353459094Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.354195092Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.354982661Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.355332937Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.356135209Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.356961434Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-12T08:42:38.421 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.357815943Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.35861277Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.359519889Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.360220679Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.360985543Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.36142164Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.361853113Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.362625176Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.363550634Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.364204309Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.364486731Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.365100958Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.365632255Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-12T08:42:38.422 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.365964849Z level=info msg="Executing migration" id="create quota table v1" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.36643364Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.367227822Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.367537348Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.370254228Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.372712663Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.375470689Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.375819791Z level=info msg="Executing migration" id="create session table" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.378475399Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.381274461Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.383969704Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.384396749Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.387225534Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.38966254Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-12T08:42:38.423 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.392490052Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-12T08:42:38.424 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.392805518Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-12T08:42:38.424 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.395638759Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-12T08:42:38.424 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.398353504Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-12T08:42:38.424 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.400951776Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-12T08:42:38.424 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.403685098Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-12T08:42:38.424 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.406468323Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-12T08:42:38.424 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.409040182Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-12T08:42:38.424 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.41146283Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-12T08:42:38.424 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.41191224Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-12T08:42:38.659 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.414385538Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.417111077Z level=info msg="Executing migration" id="create alert table v1" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.419548403Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.420060886Z level=info msg="Executing migration" id="add index alert state" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.422669391Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.42527781Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.427697794Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.428279479Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.431134778Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.433378496Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.435817298Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.436397467Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-12T08:42:38.660 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.439040082Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-12T08:42:38.661 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.44153375Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-12T08:42:38.661 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.442284079Z level=info msg="Executing migration" id="Add column is_default" 2024-04-12T08:42:38.661 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.442961584Z level=info msg="Executing migration" id="Add column frequency" 2024-04-12T08:42:38.661 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.443776793Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-12T08:42:38.661 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.444551712Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-12T08:42:38.661 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.445302688Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-12T08:42:38.662 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.445750162Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-12T08:42:38.662 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.44664731Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-12T08:42:38.662 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.44732055Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-12T08:42:38.662 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.448102116Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-12T08:42:38.662 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.448576908Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-12T08:42:38.662 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.449341033Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-12T08:42:38.662 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.450114749Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.450899549Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.451545589Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.452325301Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.453008242Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.45348359Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.453981542Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.454605426Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.454924977Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.455383161Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.455841493Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.456222254Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.45703996Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.457475541Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-12T08:42:38.663 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.458262229Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.459035604Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.459866087Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.460368817Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.461120814Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.461889219Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.462668431Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.463343442Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.464135672Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.464958675Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.467625325Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.469197311Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.472040731Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.474598288Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.477381034Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-12T08:42:38.664 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.477840779Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.48056844Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.483177845Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.485897586Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.486524859Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.489232975Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.491940399Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.494683292Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.495120983Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.497570448Z level=info msg="Executing migration" id="Move region to single row" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.49794258Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.500252448Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.50069212Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.503078211Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-12T08:42:38.665 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.503494024Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.506061526Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.508762559Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.511323658Z level=info msg="Executing migration" id="create test_data table" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.511757498Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.514404324Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.516825179Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.519342441Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.519695525Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.522318266Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.524972414Z level=info msg="Executing migration" id="create team table" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.527545186Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.52803712Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.530620086Z level=info msg="Executing migration" id="create team member table" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.533265459Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-12T08:42:38.666 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.535723613Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.536545877Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.53741855Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.538261342Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.539037338Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.539931215Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.540757199Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.541614605Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.542049034Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.542480196Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.543352026Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.544194028Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.544643022Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-12T08:42:38.667 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.545497806Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.54639147Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.547228567Z level=info msg="Executing migration" id="create tag table" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.547635644Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.548476962Z level=info msg="Executing migration" id="create login attempt table" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.549366657Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.550251559Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.550678598Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.553408756Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.556237054Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.558867991Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.559211218Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.562011614Z level=info msg="Executing migration" id="create user auth table" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.562427309Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.56486557Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.567794593Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-12T08:42:38.668 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.568548446Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-12T08:42:38.669 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.571486285Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-12T08:42:38.669 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.574291642Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-12T08:42:38.669 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.577028097Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-12T08:42:38.669 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.577474285Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-12T08:42:38.669 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.58015053Z level=info msg="Executing migration" id="create server_lock table" 2024-04-12T08:42:38.669 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.583074397Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.585839328Z level=info msg="Executing migration" id="create user auth token table" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.588679293Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.591426881Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.594096619Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.596874862Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.59765662Z level=info msg="Executing migration" id="create cache_data table" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.600358766Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.602908076Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.605648066Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.606163256Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.60884815Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.609239948Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.61154588Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-12T08:42:38.670 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.614333308Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.614860351Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.617590518Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.620423958Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.623103115Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.625822388Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.626433348Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.627675792Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.628264426Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.628667284Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.629231452Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.629790716Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.630233029Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.63050892Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.630964011Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.631354954Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.631873576Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-12T08:42:38.671 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.632375015Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.633287985Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.633711602Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.6341641Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.639057552Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.643345706Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.644183494Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.644795621Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.646062425Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.64664937Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.647239614Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.647829342Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.648403663Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.648804264Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.650049598Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-12T08:42:38.672 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.651193488Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.65240405Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.652993755Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.653588535Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.654799124Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.656097398Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.656699473Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.657855083Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.659151764Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.659588955Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.660259123Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.66085091Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.661500909Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.661918105Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-12T08:42:38.673 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.663027516Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.66572969Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.666673053Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.667552032Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.668396298Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.668725846Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.669127869Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.670002261Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.670313519Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.671232962Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.672028871Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-12T08:42:38.912 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.673016589Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-12T08:42:38.913 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.673527353Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-12T08:42:38.913 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.674098323Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-12T08:42:38.913 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.675350597Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-12T08:42:38.913 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.675773458Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-12T08:42:38.913 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.6762068Z level=info msg="Executing migration" id="create alert_image table" 2024-04-12T08:42:38.913 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.676623347Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-12T08:42:38.913 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.677035074Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-12T08:42:38.914 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.677303146Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-12T08:42:38.914 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.677699767Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-12T08:42:38.914 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.67810726Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-12T08:42:38.914 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.6781783Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.678423204Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.678796889Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.679264146Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.679385353Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.679496055Z level=info msg="alerts found to migrate" alerts=0 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.680094686Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.68094859Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.681394773Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.682247805Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.68309807Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.683934483Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.684249476Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.685062025Z level=info msg="Executing migration" id="create data_keys table" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.68549611Z level=info msg="Executing migration" id="create secrets table" 2024-04-12T08:42:38.915 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.68594519Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.690600556Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.691531007Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.692524184Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.697151094Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.701773061Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.70220362Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.703072933Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.703945874Z level=info msg="Executing migration" id="create permission table" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.704790746Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.705231752Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.705971438Z level=info msg="Executing migration" id="create role table" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.706717641Z level=info msg="Executing migration" id="add column display_name" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.707696122Z level=info msg="Executing migration" id="add column group_name" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.708613175Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-12T08:42:38.916 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.709300437Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.710067913Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.710795505Z level=info msg="Executing migration" id="create team role table" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.711191709Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.711894204Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.71261194Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.713337514Z level=info msg="Executing migration" id="create user role table" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.71376853Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.714465741Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.715212074Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.716002281Z level=info msg="Executing migration" id="create builtin role table" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.716464974Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.717185364Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.71796509Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-12T08:42:38.917 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.719049199Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.719576848Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.722442601Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.725389624Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.728163467Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.728815441Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.731697081Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.74125781Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.742994987Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.74363787Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.745533144Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.747429816Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.749356234Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.749784398Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-12T08:42:38.918 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.751682782Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.753615211Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.75552991Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.75618857Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.758162377Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.760042571Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.761926013Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.762389563Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.764284311Z level=info msg="Executing migration" id="add correlation config column" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.766321643Z level=info msg="Executing migration" id="create entity_events table" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.768151819Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.768603286Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.768691581Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.770511118Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.770623199Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-12T08:42:38.919 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.772382603Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.774272235Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.774866749Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.776789828Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.778678522Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.780559626Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.780992457Z level=info msg="Executing migration" id="Drop public config table" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.782869587Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.783304355Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.784787016Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.786664267Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.787118088Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.790167752Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-12T08:42:38.920 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.79207618Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.7939653Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.794323149Z level=info msg="Executing migration" id="add share column" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.796202456Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.798086608Z level=info msg="Executing migration" id="create file table" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.799975348Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.800439745Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.802351265Z level=info msg="Executing migration" id="create file_meta table" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.804229513Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.80610077Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.806394118Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.808355153Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-12T08:42:38.921 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.81025548Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.812115819Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.813213051Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.81510261Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.816985665Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.818925061Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.819566706Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.821462044Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.823334646Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.825281116Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.825915904Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.835644637Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.837659508Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-12T08:42:38.922 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.839448759Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-12T08:42:38.923 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.842946412Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-12T08:42:38.923 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.844805091Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-12T08:42:38.923 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.846715245Z level=info msg="Executing migration" id="create folder table" 2024-04-12T08:42:38.923 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.848576038Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-12T08:42:38.923 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.849009311Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-12T08:42:38.923 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.850903206Z level=info msg="Executing migration" id="Update folder title length" 2024-04-12T08:42:38.923 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.852702425Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-12T08:42:38.923 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=migrator t=2024-04-12T08:42:38.85467185Z level=info msg="migrations completed" performed=485 skipped=0 duration=624.100859ms 2024-04-12T08:42:38.923 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=sqlstore t=2024-04-12T08:42:38.855239964Z level=info msg="Created default organization" 2024-04-12T08:42:38.923 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=plugin.loader t=2024-04-12T08:42:38.911664195Z level=info msg="Plugin registered" pluginID=input 2024-04-12T08:42:39.198 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=plugin.loader t=2024-04-12T08:42:38.939049952Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-12T08:42:39.198 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=plugin.loader t=2024-04-12T08:42:38.939069718Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-12T08:42:39.198 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=secrets t=2024-04-12T08:42:38.939102367Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-12T08:42:39.198 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=query_data t=2024-04-12T08:42:38.940779562Z level=info msg="Query Service initialization" 2024-04-12T08:42:39.198 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=live.push_http t=2024-04-12T08:42:38.943193517Z level=info msg="Live Push Gateway initialization" 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[40003]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.337 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:42:39.338 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:39 smithi090 ceph-mon[29100]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:39.462 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.462 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.462 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.463 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:39.463 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:42:39.463 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-mon[33573]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:40.033 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=infra.usagestats.collector t=2024-04-12T08:42:39.776733678Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-12T08:42:40.033 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=provisioning.datasources t=2024-04-12T08:42:39.7770359Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-12T08:42:40.033 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=provisioning.datasources t=2024-04-12T08:42:39.810708373Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-12T08:42:40.033 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=provisioning.datasources t=2024-04-12T08:42:39.851750426Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-12T08:42:40.033 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=provisioning.alerting t=2024-04-12T08:42:39.876271412Z level=info msg="starting to provision alerting" 2024-04-12T08:42:40.033 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=provisioning.alerting t=2024-04-12T08:42:39.876299629Z level=info msg="finished to provision alerting" 2024-04-12T08:42:40.033 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=ngalert.state.manager t=2024-04-12T08:42:39.87645396Z level=info msg="Warming state cache for startup" 2024-04-12T08:42:40.033 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=grafanaStorageLogger t=2024-04-12T08:42:39.877004355Z level=info msg="storage starting" 2024-04-12T08:42:40.033 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=http.server t=2024-04-12T08:42:39.883351926Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-12T08:42:40.033 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=ngalert.state.manager t=2024-04-12T08:42:39.888047965Z level=info msg="State cache has been initialized" states=0 duration=11.592967ms 2024-04-12T08:42:40.034 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=ngalert.multiorg.alertmanager t=2024-04-12T08:42:39.888103894Z level=info msg="starting MultiOrg Alertmanager" 2024-04-12T08:42:40.034 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=ticker t=2024-04-12T08:42:39.88811903Z level=info msg=starting first_tick=2024-04-12T08:42:40Z 2024-04-12T08:42:40.034 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:42:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[62809]: logger=sqlstore.transactions t=2024-04-12T08:42:39.934777417Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-12T08:42:41.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:41 smithi114 ceph-mon[33573]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:41.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:41 smithi090 ceph-mon[40003]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:41.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:41 smithi090 ceph-mon[29100]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:42.573 INFO:teuthology.orchestra.run.smithi090.stdout:236223201295 2024-04-12T08:42:42.573 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-12T08:42:43.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:42 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:43.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:42 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:43.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:42 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:43.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:42 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:43.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:42 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:43.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:42 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:43.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:42 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:43.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:42 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:43.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:42 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:42:44.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:43 smithi090 ceph-mon[40003]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:43 smithi090 ceph-mon[29100]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:44.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:43 smithi114 ceph-mon[33573]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:45.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:45 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:42:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:42:45.579 INFO:teuthology.orchestra.run.smithi090.stdout:201863462931 2024-04-12T08:42:45.579 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-12T08:42:45.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:45 smithi090 ceph-mon[29100]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:45.997 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:45 smithi090 ceph-mon[40003]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:45 smithi114 ceph-mon[33573]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:47.148 INFO:teuthology.orchestra.run.smithi090.stdout:77309411368 2024-04-12T08:42:47.149 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-12T08:42:48.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:47 smithi090 ceph-mon[29100]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:48.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:47 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:48.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:47 smithi090 ceph-mon[40003]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:48.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:47 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:48.230 INFO:teuthology.orchestra.run.smithi090.stdout:55834574893 2024-04-12T08:42:48.230 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-12T08:42:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:47 smithi114 ceph-mon[33573]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:47 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:42:48.749 INFO:teuthology.orchestra.run.smithi090.stdout:167503724568 2024-04-12T08:42:48.750 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-12T08:42:49.475 INFO:teuthology.orchestra.run.smithi090.stdout:111669149731 2024-04-12T08:42:49.475 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-12T08:42:50.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:49 smithi090 ceph-mon[40003]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:50.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:49 smithi090 ceph-mon[29100]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:50.248 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:49 smithi114 ceph-mon[33573]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:50.588 INFO:teuthology.orchestra.run.smithi090.stdout:34359738419 2024-04-12T08:42:50.588 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-12T08:42:51.084 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:51.084 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:51.095 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:51.567 INFO:teuthology.orchestra.run.smithi090.stdout:133143986207 2024-04-12T08:42:51.567 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-12T08:42:52.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:51 smithi090 ceph-mon[40003]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:52.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:51 smithi090 ceph-mon[29100]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:51 smithi114 ceph-mon[33573]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:52.477 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:52.478 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:52.478 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:54.262 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:53 smithi090 ceph-mon[40003]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:54.262 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:53 smithi090 ceph-mon[29100]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:54.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:53 smithi114 ceph-mon[33573]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:55.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:42:55 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:42:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:42:55.841 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:42:56.213 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:55 smithi090 ceph-mon[40003]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:56.213 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:55 smithi090 ceph-mon[29100]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:55 smithi114 ceph-mon[33573]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:58.121 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:57 smithi090 ceph-mon[40003]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:58.122 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:57 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1844051112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T08:42:58.122 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:57 smithi090 ceph-mon[29100]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:58.122 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:57 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1844051112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T08:42:58.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:57 smithi114 ceph-mon[33573]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:42:58.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:57 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1844051112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T08:42:59.886 INFO:teuthology.orchestra.run.smithi090.stdout:111669149734 2024-04-12T08:43:00.116 INFO:teuthology.orchestra.run.smithi090.stdout:167503724572 2024-04-12T08:43:00.116 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:59 smithi090 ceph-mon[40003]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:00.116 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:42:59 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2711809145' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T08:43:00.116 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:59 smithi090 ceph-mon[29100]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:00.117 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:42:59 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2711809145' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T08:43:00.216 INFO:teuthology.orchestra.run.smithi090.stdout:201863462934 2024-04-12T08:43:00.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:59 smithi114 ceph-mon[33573]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:00.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:42:59 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2711809145' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T08:43:00.406 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:00.813 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462931 got 201863462934 for osd.6 2024-04-12T08:43:00.813 DEBUG:teuthology.parallel:result is None 2024-04-12T08:43:01.133 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:00 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/292212717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T08:43:01.133 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:00 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/292212717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T08:43:01.205 INFO:teuthology.orchestra.run.smithi090.stdout:236223201299 2024-04-12T08:43:01.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:00 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/292212717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T08:43:01.848 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201295 got 236223201299 for osd.7 2024-04-12T08:43:01.848 DEBUG:teuthology.parallel:result is None 2024-04-12T08:43:02.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:01 smithi090 ceph-mon[40003]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:02.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:01 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4281336205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T08:43:02.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:01 smithi090 ceph-mon[29100]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:02.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:01 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4281336205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T08:43:02.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:01 smithi114 ceph-mon[33573]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:02.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:01 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4281336205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T08:43:03.254 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:03.255 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:03.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:03.579 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149731 got 111669149734 for osd.3 2024-04-12T08:43:03.579 DEBUG:teuthology.parallel:result is None 2024-04-12T08:43:04.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:03 smithi090 ceph-mon[40003]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:03 smithi090 ceph-mon[29100]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:04.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:03 smithi114 ceph-mon[33573]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:04.504 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724568 got 167503724572 for osd.5 2024-04-12T08:43:04.504 DEBUG:teuthology.parallel:result is None 2024-04-12T08:43:05.135 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:04 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3061949789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T08:43:05.136 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:04 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3061949789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T08:43:05.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:04 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3061949789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T08:43:05.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:43:05 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:43:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:43:05.555 INFO:teuthology.orchestra.run.smithi090.stdout:55834574896 2024-04-12T08:43:05.628 INFO:teuthology.orchestra.run.smithi090.stdout:77309411372 2024-04-12T08:43:06.228 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:05 smithi090 ceph-mon[40003]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:06.228 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:05 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2206263304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T08:43:06.228 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:05 smithi090 ceph-mon[29100]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:06.228 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:05 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2206263304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T08:43:06.229 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574893 got 55834574896 for osd.1 2024-04-12T08:43:06.229 DEBUG:teuthology.parallel:result is None 2024-04-12T08:43:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:05 smithi114 ceph-mon[33573]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:05 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2206263304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T08:43:07.654 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411368 got 77309411372 for osd.2 2024-04-12T08:43:07.654 DEBUG:teuthology.parallel:result is None 2024-04-12T08:43:08.068 INFO:teuthology.orchestra.run.smithi090.stdout:34359738422 2024-04-12T08:43:08.313 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:07 smithi090 ceph-mon[29100]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:08.314 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:07 smithi090 ceph-mon[40003]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:08.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:07 smithi114 ceph-mon[33573]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:09.354 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:08 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1900265799' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T08:43:09.355 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:08 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1900265799' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T08:43:09.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:08 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1900265799' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T08:43:09.886 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738419 got 34359738422 for osd.0 2024-04-12T08:43:09.886 DEBUG:teuthology.parallel:result is None 2024-04-12T08:43:10.325 INFO:teuthology.orchestra.run.smithi090.stdout:133143986211 2024-04-12T08:43:10.326 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:09 smithi090 ceph-mon[40003]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:10.326 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:09 smithi090 ceph-mon[29100]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:10.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:09 smithi114 ceph-mon[33573]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:10.975 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:10 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1004602408' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T08:43:10.975 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:10 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1004602408' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T08:43:11.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:10 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1004602408' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T08:43:11.504 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986207 got 133143986211 for osd.4 2024-04-12T08:43:11.505 DEBUG:teuthology.parallel:result is None 2024-04-12T08:43:11.505 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-12T08:43:11.505 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T08:43:11.779 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[40003]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[29100]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:12.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:12.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:12.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:12.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:12 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:12 smithi114 ceph-mon[33573]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:12 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:12 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:12 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:12 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:12 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:13.152 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:13 smithi090 ceph-mon[40003]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-12T08:43:13.152 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:13 smithi090 ceph-mon[40003]: Reconfiguring daemon alertmanager.a on smithi090 2024-04-12T08:43:13.153 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:13 smithi090 ceph-mon[29100]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-12T08:43:13.153 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:13 smithi090 ceph-mon[29100]: Reconfiguring daemon alertmanager.a on smithi090 2024-04-12T08:43:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:13 smithi114 ceph-mon[33573]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-12T08:43:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:13 smithi114 ceph-mon[33573]: Reconfiguring daemon alertmanager.a on smithi090 2024-04-12T08:43:14.254 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:14 smithi090 ceph-mon[40003]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:14.254 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:14 smithi090 ceph-mon[29100]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:14 smithi114 ceph-mon[33573]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:14.459 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:43:14.460 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-12T08:43:15.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:43:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:43:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:43:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:16 smithi114 ceph-mon[33573]: from='client.14601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:43:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:16 smithi114 ceph-mon[33573]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:16.402 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:16 smithi090 ceph-mon[40003]: from='client.14601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:43:16.402 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:16 smithi090 ceph-mon[40003]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:16.402 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:16 smithi090 ceph-mon[29100]: from='client.14601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:43:16.402 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:16 smithi090 ceph-mon[29100]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:16.467 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2024-04-12T08:43:12.610522+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385564,"kb_used_data":6704,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575652,"statfs":{"total":767960285184,"available":765517467648,"internally_reserved":0,"allocated":6864896,"data_stored":3897469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001649"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:20.481334+0000","last_change":"2024-04-12T08:42:18.340361+0000","last_active":"2024-04-12T08:42:20.481334+0000","last_peered":"2024-04-12T08:42:20.481334+0000","last_clean":"2024-04-12T08:42:20.481334+0000","last_became_active":"2024-04-12T08:42:18.340050+0000","last_became_peered":"2024-04-12T08:42:18.340050+0000","last_unstale":"2024-04-12T08:42:20.481334+0000","last_undegraded":"2024-04-12T08:42:20.481334+0000","last_fullsized":"2024-04-12T08:42:20.481334+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:56:18.913365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:20.914054+0000","last_change":"2024-04-12T08:42:18.728964+0000","last_active":"2024-04-12T08:42:20.914054+0000","last_peered":"2024-04-12T08:42:20.914054+0000","last_clean":"2024-04-12T08:42:20.914054+0000","last_became_active":"2024-04-12T08:42:18.728592+0000","last_became_peered":"2024-04-12T08:42:18.728592+0000","last_unstale":"2024-04-12T08:42:20.914054+0000","last_undegraded":"2024-04-12T08:42:20.914054+0000","last_fullsized":"2024-04-12T08:42:20.914054+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:54:31.647606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:19.350788+0000","last_change":"2024-04-12T08:42:18.728907+0000","last_active":"2024-04-12T08:42:19.350788+0000","last_peered":"2024-04-12T08:42:19.350788+0000","last_clean":"2024-04-12T08:42:19.350788+0000","last_became_active":"2024-04-12T08:42:18.728438+0000","last_became_peered":"2024-04-12T08:42:18.728438+0000","last_unstale":"2024-04-12T08:42:19.350788+0000","last_undegraded":"2024-04-12T08:42:19.350788+0000","last_fullsized":"2024-04-12T08:42:19.350788+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:20:28.071214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:19.350732+0000","last_change":"2024-04-12T08:42:18.346705+0000","last_active":"2024-04-12T08:42:19.350732+0000","last_peered":"2024-04-12T08:42:19.350732+0000","last_clean":"2024-04-12T08:42:19.350732+0000","last_became_active":"2024-04-12T08:42:18.346382+0000","last_became_peered":"2024-04-12T08:42:18.346382+0000","last_unstale":"2024-04-12T08:42:19.350732+0000","last_undegraded":"2024-04-12T08:42:19.350732+0000","last_fullsized":"2024-04-12T08:42:19.350732+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:12:09.463598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:19.350780+0000","last_change":"2024-04-12T08:41:37.324892+0000","last_active":"2024-04-12T08:42:19.350780+0000","last_peered":"2024-04-12T08:42:19.350780+0000","last_clean":"2024-04-12T08:42:19.350780+0000","last_became_active":"2024-04-12T08:41:37.324471+0000","last_became_peered":"2024-04-12T08:41:37.324471+0000","last_unstale":"2024-04-12T08:42:19.350780+0000","last_undegraded":"2024-04-12T08:42:19.350780+0000","last_fullsized":"2024-04-12T08:42:19.350780+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_clean_scrub_stamp":"2024-04-12T08:39:39.156733+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:03:42.742838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:20.914250+0000","last_change":"2024-04-12T08:42:18.339523+0000","last_active":"2024-04-12T08:42:20.914250+0000","last_peered":"2024-04-12T08:42:20.914250+0000","last_clean":"2024-04-12T08:42:20.914250+0000","last_became_active":"2024-04-12T08:42:18.339215+0000","last_became_peered":"2024-04-12T08:42:18.339215+0000","last_unstale":"2024-04-12T08:42:20.914250+0000","last_undegraded":"2024-04-12T08:42:20.914250+0000","last_fullsized":"2024-04-12T08:42:20.914250+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:45:58.963209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:19.352590+0000","last_change":"2024-04-12T08:42:18.457898+0000","last_active":"2024-04-12T08:42:19.352590+0000","last_peered":"2024-04-12T08:42:19.352590+0000","last_clean":"2024-04-12T08:42:19.352590+0000","last_became_active":"2024-04-12T08:42:18.457597+0000","last_became_peered":"2024-04-12T08:42:18.457597+0000","last_unstale":"2024-04-12T08:42:19.352590+0000","last_undegraded":"2024-04-12T08:42:19.352590+0000","last_fullsized":"2024-04-12T08:42:19.352590+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:01:02.172496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:20.914212+0000","last_change":"2024-04-12T08:42:18.340485+0000","last_active":"2024-04-12T08:42:20.914212+0000","last_peered":"2024-04-12T08:42:20.914212+0000","last_clean":"2024-04-12T08:42:20.914212+0000","last_became_active":"2024-04-12T08:42:18.340137+0000","last_became_peered":"2024-04-12T08:42:18.340137+0000","last_unstale":"2024-04-12T08:42:20.914212+0000","last_undegraded":"2024-04-12T08:42:20.914212+0000","last_fullsized":"2024-04-12T08:42:20.914212+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:05:14.634360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:20.914137+0000","last_change":"2024-04-12T08:42:18.346660+0000","last_active":"2024-04-12T08:42:20.914137+0000","last_peered":"2024-04-12T08:42:20.914137+0000","last_clean":"2024-04-12T08:42:20.914137+0000","last_became_active":"2024-04-12T08:42:18.346313+0000","last_became_peered":"2024-04-12T08:42:18.346313+0000","last_unstale":"2024-04-12T08:42:20.914137+0000","last_undegraded":"2024-04-12T08:42:20.914137+0000","last_fullsized":"2024-04-12T08:42:20.914137+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:40.908045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.35199999999999998}]},{"osd":1,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.39900000000000002}]},{"osd":2,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.53300000000000003}]},{"osd":3,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.432},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.60099999999999998}]},{"osd":4,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.5}]},{"osd":5,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.51900000000000002}]},{"osd":6,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.58299999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":855557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:12 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.59299999999999997}]},{"osd":1,"last update":"Fri Apr 12 08:42:12 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.52500000000000002}]},{"osd":2,"last update":"Fri Apr 12 08:42:12 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.40200000000000002}]},{"osd":3,"last update":"Fri Apr 12 08:42:12 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.51100000000000001}]},{"osd":4,"last update":"Fri Apr 12 08:42:12 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.47699999999999998}]},{"osd":5,"last update":"Fri Apr 12 08:42:12 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.443},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.41299999999999998}]},{"osd":7,"last update":"Fri Apr 12 08:42:35 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.498}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":265512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:17 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.34799999999999998,"15min":0.33200000000000002},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.45900000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.35899999999999999,"15min":0.33700000000000002},"min":{"1min":0.23899999999999999,"5min":0.219,"15min":0.219},"max":{"1min":0.52100000000000002,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.32900000000000001}]},{"osd":2,"last update":"Fri Apr 12 08:42:43 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.41199999999999998,"15min":0.42099999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.60499999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.44500000000000001,"15min":0.45300000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.60699999999999998,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.39100000000000001}]},{"osd":3,"last update":"Fri Apr 12 08:43:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.46700000000000003,"15min":0.48599999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.58099999999999996,"5min":0.628,"15min":0.628},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46800000000000003},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.316}]},{"osd":4,"last update":"Fri Apr 12 08:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55400000000000005,"15min":0.55900000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.69099999999999995,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.73999999999999999}]},{"osd":5,"last update":"Fri Apr 12 08:42:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.55600000000000005,"15min":0.55400000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.78000000000000003}]},{"osd":6,"last update":"Fri Apr 12 08:42:10 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.75700000000000001}]},{"osd":7,"last update":"Fri Apr 12 08:42:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.59099999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 08:42:19 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.33100000000000002,"15min":0.307},"min":{"1min":0.214,"5min":0.189,"15min":0.189},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.32200000000000001},"min":{"1min":0.26300000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.316}]},{"osd":2,"last update":"Fri Apr 12 08:42:45 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.38600000000000001,"15min":0.38100000000000001},"min":{"1min":0.26300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.375,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.33900000000000002}]},{"osd":3,"last update":"Fri Apr 12 08:43:04 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.438,"15min":0.45300000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.50800000000000001,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.46400000000000002,"15min":0.47999999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.63500000000000001,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.39700000000000002}]},{"osd":4,"last update":"Fri Apr 12 08:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58599999999999997,"15min":0.58799999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59999999999999998,"15min":0.60699999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.53300000000000003}]},{"osd":5,"last update":"Fri Apr 12 08:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.59899999999999998,"15min":0.60999999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.72899999999999998,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.61799999999999999,"15min":0.624},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.56100000000000005}]},{"osd":6,"last update":"Fri Apr 12 08:42:14 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.54800000000000004}]},{"osd":7,"last update":"Fri Apr 12 08:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.61499999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:44 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.36499999999999999,"15min":0.35899999999999999},"min":{"1min":0.28199999999999997,"5min":0.218,"15min":0.218},"max":{"1min":0.48999999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.39500000000000002,"15min":0.38900000000000001},"min":{"1min":0.252,"5min":0.248,"15min":0.248},"max":{"1min":0.49299999999999999,"5min":0.629,"15min":0.629},"last":0.58799999999999997}]},{"osd":1,"last update":"Fri Apr 12 08:42:44 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.38,"15min":0.371},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.56299999999999994,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.5},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41099999999999998,"15min":0.39400000000000002},"min":{"1min":0.34499999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.53300000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.65400000000000003}]},{"osd":3,"last update":"Fri Apr 12 08:43:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.46999999999999997,"15min":0.48399999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.58099999999999996,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.45700000000000002,"15min":0.47099999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57199999999999995,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.53300000000000003}]},{"osd":4,"last update":"Fri Apr 12 08:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":1.2210000000000001,"15min":1.335},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.82099999999999995,"5min":14.991,"15min":14.991},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.59399999999999997,"15min":0.59799999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.74399999999999999,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.77600000000000002}]},{"osd":5,"last update":"Fri Apr 12 08:42:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.65200000000000002,"15min":0.66300000000000003},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.78300000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.63500000000000001,"15min":0.64500000000000002},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.75600000000000001}]},{"osd":6,"last update":"Fri Apr 12 08:42:11 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.68999999999999995}]},{"osd":7,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.746}]}]},{"osd":3,"up_from":26,"seq":111669149736,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:05 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.38900000000000001,"15min":0.39200000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.434,"15min":0.438},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.28599999999999998}]},{"osd":1,"last update":"Fri Apr 12 08:42:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.45300000000000001,"15min":0.46100000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.62,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.46700000000000003,"15min":0.47699999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63400000000000001,"5min":0.751,"15min":0.751},"last":0.25700000000000001}]},{"osd":2,"last update":"Fri Apr 12 08:42:05 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.48699999999999999,"15min":0.496},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.64900000000000002,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.317},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.45200000000000001,"15min":0.45800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.36699999999999999}]},{"osd":4,"last update":"Fri Apr 12 08:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.59099999999999997,"15min":0.59699999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.59199999999999997,"15min":0.59999999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71599999999999997,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.497}]},{"osd":5,"last update":"Fri Apr 12 08:42:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.59699999999999998,"15min":0.60199999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.622},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.60199999999999998,"15min":0.60999999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.76200000000000001,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.54800000000000004}]},{"osd":6,"last update":"Fri Apr 12 08:42:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.52200000000000002}]},{"osd":7,"last update":"Fri Apr 12 08:42:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.57199999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.42399999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.443},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72899999999999998,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.50900000000000001}]},{"osd":1,"last update":"Fri Apr 12 08:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.41399999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.69999999999999996,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.54100000000000004}]},{"osd":2,"last update":"Fri Apr 12 08:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.54500000000000004,"15min":0.54200000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79800000000000004,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.65300000000000002}]},{"osd":3,"last update":"Fri Apr 12 08:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39700000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76800000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.63}]},{"osd":5,"last update":"Fri Apr 12 08:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.38300000000000001,"15min":0.377},"min":{"1min":0.30499999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.42699999999999999,"15min":0.42099999999999999},"min":{"1min":0.33500000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.73899999999999999,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.74099999999999999}]},{"osd":6,"last update":"Fri Apr 12 08:42:10 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.64300000000000002}]},{"osd":7,"last update":"Fri Apr 12 08:42:35 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.56999999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.52600000000000002,"15min":0.53100000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.52800000000000002,"15min":0.52600000000000002},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.38800000000000001}]},{"osd":1,"last update":"Fri Apr 12 08:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.52100000000000002,"15min":0.52600000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.41499999999999998}]},{"osd":2,"last update":"Fri Apr 12 08:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.52900000000000003,"15min":0.53200000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.55100000000000005,"15min":0.55700000000000005},"min":{"1min":0.39900000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.48199999999999998}]},{"osd":3,"last update":"Fri Apr 12 08:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52000000000000002,"15min":0.52200000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56399999999999995,"15min":0.56499999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.435}]},{"osd":4,"last update":"Fri Apr 12 08:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39700000000000002,"15min":0.39200000000000002},"min":{"1min":0.312,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.317},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.40799999999999997,"15min":0.40000000000000002},"min":{"1min":0.29799999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.42399999999999999}]},{"osd":6,"last update":"Fri Apr 12 08:42:09 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.373},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.34499999999999997}]},{"osd":7,"last update":"Fri Apr 12 08:42:36 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.44700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T08:43:16.469 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T08:43:17.165 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:17.286 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:17 smithi090 systemd[1]: Stopping Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:43:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:18 smithi114 ceph-mon[33573]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:18 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:18.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:18 smithi090 ceph-mon[40003]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:18.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:18 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:18.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:17 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[68599]: ts=2024-04-12T08:43:17.985Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-12T08:43:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:18 smithi090 ceph-mon[29100]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:18 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:18.806 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:18 smithi090 podman[73104]: 2024-04-12 08:43:18.535289238 +0000 UTC m=+1.004632593 container died 3d4f8e008936ca7af0c21c8e7737572bfe02776150eed319f6c29c7ac03b03a2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:19.150 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:18 smithi090 podman[73104]: 2024-04-12 08:43:18.914175493 +0000 UTC m=+1.383518847 container cleanup 3d4f8e008936ca7af0c21c8e7737572bfe02776150eed319f6c29c7ac03b03a2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:19.150 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:18 smithi090 bash[73104]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a 2024-04-12T08:43:19.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:19 smithi090 podman[73132]: 2024-04-12 08:43:19.150037622 +0000 UTC m=+1.139813749 container remove 3d4f8e008936ca7af0c21c8e7737572bfe02776150eed319f6c29c7ac03b03a2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:19.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:19 smithi090 podman[73132]: 2024-04-12 08:43:19.225760815 +0000 UTC m=+1.215536940 volume remove c64f2682eb342d21a6a665566292d371c60931a64b385efea6cb6bf0119719df 2024-04-12T08:43:19.895 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:43:19.896 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:19 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-12T08:43:19.896 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:19 smithi090 systemd[1]: Stopped Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:43:19.896 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:19 smithi090 systemd[1]: Starting Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:43:19.897 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-12T08:43:20.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:20 smithi090 ceph-mon[40003]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:20.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:20 smithi090 ceph-mon[29100]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:20.173 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:19 smithi090 podman[73273]: 2024-04-12 08:43:19.828354924 +0000 UTC m=+0.017608563 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-12T08:43:20.173 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:19 smithi090 podman[73273]: 2024-04-12 08:43:19.930796907 +0000 UTC m=+0.120050532 volume create 09e6f45e6bbba25d243daf60d1a660b8346bbe53a4aaf0bb2926f86cb9c5f48d 2024-04-12T08:43:20.173 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 podman[73273]: 2024-04-12 08:43:20.039650734 +0000 UTC m=+0.228904406 container create 0893314a69712e380d603b3415ef7bf447ee0f3e9373b8b45246d4126cff7312 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:20.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:20 smithi114 ceph-mon[33573]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:20.503 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 podman[73273]: 2024-04-12 08:43:20.377586361 +0000 UTC m=+0.566839987 container init 0893314a69712e380d603b3415ef7bf447ee0f3e9373b8b45246d4126cff7312 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:20.503 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 podman[73273]: 2024-04-12 08:43:20.380604801 +0000 UTC m=+0.569858436 container start 0893314a69712e380d603b3415ef7bf447ee0f3e9373b8b45246d4126cff7312 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:20.503 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:43:20.392Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-12T08:43:20.503 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:43:20.392Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-12T08:43:20.503 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:43:20.393Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.90 port=9094 2024-04-12T08:43:20.503 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:43:20.394Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-12T08:43:20.503 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:43:20.417Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-12T08:43:20.503 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:43:20.418Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-12T08:43:20.503 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:43:20.420Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-12T08:43:20.503 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:43:20.420Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-12T08:43:20.823 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 bash[73273]: 0893314a69712e380d603b3415ef7bf447ee0f3e9373b8b45246d4126cff7312 2024-04-12T08:43:20.823 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:20 smithi090 systemd[1]: Started Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:43:21.157 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:21 smithi090 ceph-mon[40003]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:43:21.158 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:21.158 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:21.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:21 smithi090 ceph-mon[29100]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:43:21.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:21.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:21.183 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2024-04-12T08:43:18.611478+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385564,"kb_used_data":6704,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575652,"statfs":{"total":767960285184,"available":765517467648,"internally_reserved":0,"allocated":6864896,"data_stored":3897469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001747"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:20.481334+0000","last_change":"2024-04-12T08:42:18.340361+0000","last_active":"2024-04-12T08:42:20.481334+0000","last_peered":"2024-04-12T08:42:20.481334+0000","last_clean":"2024-04-12T08:42:20.481334+0000","last_became_active":"2024-04-12T08:42:18.340050+0000","last_became_peered":"2024-04-12T08:42:18.340050+0000","last_unstale":"2024-04-12T08:42:20.481334+0000","last_undegraded":"2024-04-12T08:42:20.481334+0000","last_fullsized":"2024-04-12T08:42:20.481334+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:56:18.913365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:20.914054+0000","last_change":"2024-04-12T08:42:18.728964+0000","last_active":"2024-04-12T08:42:20.914054+0000","last_peered":"2024-04-12T08:42:20.914054+0000","last_clean":"2024-04-12T08:42:20.914054+0000","last_became_active":"2024-04-12T08:42:18.728592+0000","last_became_peered":"2024-04-12T08:42:18.728592+0000","last_unstale":"2024-04-12T08:42:20.914054+0000","last_undegraded":"2024-04-12T08:42:20.914054+0000","last_fullsized":"2024-04-12T08:42:20.914054+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:54:31.647606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:19.350788+0000","last_change":"2024-04-12T08:42:18.728907+0000","last_active":"2024-04-12T08:42:19.350788+0000","last_peered":"2024-04-12T08:42:19.350788+0000","last_clean":"2024-04-12T08:42:19.350788+0000","last_became_active":"2024-04-12T08:42:18.728438+0000","last_became_peered":"2024-04-12T08:42:18.728438+0000","last_unstale":"2024-04-12T08:42:19.350788+0000","last_undegraded":"2024-04-12T08:42:19.350788+0000","last_fullsized":"2024-04-12T08:42:19.350788+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:20:28.071214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:19.350732+0000","last_change":"2024-04-12T08:42:18.346705+0000","last_active":"2024-04-12T08:42:19.350732+0000","last_peered":"2024-04-12T08:42:19.350732+0000","last_clean":"2024-04-12T08:42:19.350732+0000","last_became_active":"2024-04-12T08:42:18.346382+0000","last_became_peered":"2024-04-12T08:42:18.346382+0000","last_unstale":"2024-04-12T08:42:19.350732+0000","last_undegraded":"2024-04-12T08:42:19.350732+0000","last_fullsized":"2024-04-12T08:42:19.350732+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:12:09.463598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:19.350780+0000","last_change":"2024-04-12T08:41:37.324892+0000","last_active":"2024-04-12T08:42:19.350780+0000","last_peered":"2024-04-12T08:42:19.350780+0000","last_clean":"2024-04-12T08:42:19.350780+0000","last_became_active":"2024-04-12T08:41:37.324471+0000","last_became_peered":"2024-04-12T08:41:37.324471+0000","last_unstale":"2024-04-12T08:42:19.350780+0000","last_undegraded":"2024-04-12T08:42:19.350780+0000","last_fullsized":"2024-04-12T08:42:19.350780+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_clean_scrub_stamp":"2024-04-12T08:39:39.156733+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:03:42.742838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:20.914250+0000","last_change":"2024-04-12T08:42:18.339523+0000","last_active":"2024-04-12T08:42:20.914250+0000","last_peered":"2024-04-12T08:42:20.914250+0000","last_clean":"2024-04-12T08:42:20.914250+0000","last_became_active":"2024-04-12T08:42:18.339215+0000","last_became_peered":"2024-04-12T08:42:18.339215+0000","last_unstale":"2024-04-12T08:42:20.914250+0000","last_undegraded":"2024-04-12T08:42:20.914250+0000","last_fullsized":"2024-04-12T08:42:20.914250+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:45:58.963209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:19.352590+0000","last_change":"2024-04-12T08:42:18.457898+0000","last_active":"2024-04-12T08:42:19.352590+0000","last_peered":"2024-04-12T08:42:19.352590+0000","last_clean":"2024-04-12T08:42:19.352590+0000","last_became_active":"2024-04-12T08:42:18.457597+0000","last_became_peered":"2024-04-12T08:42:18.457597+0000","last_unstale":"2024-04-12T08:42:19.352590+0000","last_undegraded":"2024-04-12T08:42:19.352590+0000","last_fullsized":"2024-04-12T08:42:19.352590+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:01:02.172496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:20.914212+0000","last_change":"2024-04-12T08:42:18.340485+0000","last_active":"2024-04-12T08:42:20.914212+0000","last_peered":"2024-04-12T08:42:20.914212+0000","last_clean":"2024-04-12T08:42:20.914212+0000","last_became_active":"2024-04-12T08:42:18.340137+0000","last_became_peered":"2024-04-12T08:42:18.340137+0000","last_unstale":"2024-04-12T08:42:20.914212+0000","last_undegraded":"2024-04-12T08:42:20.914212+0000","last_fullsized":"2024-04-12T08:42:20.914212+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:05:14.634360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T08:42:20.914137+0000","last_change":"2024-04-12T08:42:18.346660+0000","last_active":"2024-04-12T08:42:20.914137+0000","last_peered":"2024-04-12T08:42:20.914137+0000","last_clean":"2024-04-12T08:42:20.914137+0000","last_became_active":"2024-04-12T08:42:18.346313+0000","last_became_peered":"2024-04-12T08:42:18.346313+0000","last_unstale":"2024-04-12T08:42:20.914137+0000","last_undegraded":"2024-04-12T08:42:20.914137+0000","last_fullsized":"2024-04-12T08:42:20.914137+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:40.908045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201303,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.60499999999999998}]},{"osd":1,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.62},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.501}]},{"osd":2,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.58599999999999997}]},{"osd":3,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.53600000000000003}]},{"osd":4,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.47599999999999998}]},{"osd":5,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.64100000000000001}]},{"osd":6,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.63100000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462939,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":855557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.51500000000000001,"15min":0.51300000000000001},"min":{"1min":0.377,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67600000000000005,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.53000000000000003,"15min":0.52600000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.61099999999999999}]},{"osd":1,"last update":"Fri Apr 12 08:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.41299999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.77100000000000002,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.503},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.55800000000000005,"15min":0.55700000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.65800000000000003}]},{"osd":2,"last update":"Fri Apr 12 08:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.54500000000000004,"15min":0.54200000000000004},"min":{"1min":0.44800000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.67200000000000004,"5min":0.753,"15min":0.753},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.73899999999999999,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.72999999999999998}]},{"osd":3,"last update":"Fri Apr 12 08:43:15 2024","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":0.63300000000000001,"15min":0.56100000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":10.471,"5min":10.471,"15min":10.471},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.55600000000000005}]},{"osd":4,"last update":"Fri Apr 12 08:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.434,"15min":0.43099999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.626},{"interface":"front","average":{"1min":0.501,"5min":0.45600000000000002,"15min":0.44800000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72199999999999998,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.57399999999999995}]},{"osd":5,"last update":"Fri Apr 12 08:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.46999999999999997,"15min":0.46800000000000003},"min":{"1min":0.32800000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.48199999999999998,"15min":0.47399999999999998},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.745}]},{"osd":7,"last update":"Fri Apr 12 08:42:35 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.76800000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574900,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":265512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:17 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.34799999999999998,"15min":0.33200000000000002},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.45900000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.35899999999999999,"15min":0.33700000000000002},"min":{"1min":0.23899999999999999,"5min":0.219,"15min":0.219},"max":{"1min":0.52100000000000002,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.495}]},{"osd":2,"last update":"Fri Apr 12 08:42:43 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.41199999999999998,"15min":0.42099999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.60499999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.44500000000000001,"15min":0.45300000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.60699999999999998,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.39200000000000002}]},{"osd":3,"last update":"Fri Apr 12 08:43:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.46700000000000003,"15min":0.48599999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.58099999999999996,"5min":0.628,"15min":0.628},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46800000000000003},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.442}]},{"osd":4,"last update":"Fri Apr 12 08:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55400000000000005,"15min":0.55900000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.69099999999999995,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.73299999999999998}]},{"osd":5,"last update":"Fri Apr 12 08:42:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.55600000000000005,"15min":0.55400000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.746}]},{"osd":6,"last update":"Fri Apr 12 08:43:10 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.57099999999999995,"15min":0.56399999999999995},"min":{"1min":0.40999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.76400000000000001}]},{"osd":7,"last update":"Fri Apr 12 08:42:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.623}]}]},{"osd":0,"up_from":8,"seq":34359738425,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 08:42:19 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.33100000000000002,"15min":0.307},"min":{"1min":0.214,"5min":0.189,"15min":0.189},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.378},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.32200000000000001},"min":{"1min":0.26300000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.41899999999999998}]},{"osd":2,"last update":"Fri Apr 12 08:42:45 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.38600000000000001,"15min":0.38100000000000001},"min":{"1min":0.26300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.375,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.40400000000000003}]},{"osd":3,"last update":"Fri Apr 12 08:43:04 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.438,"15min":0.45300000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.50800000000000001,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.46400000000000002,"15min":0.47999999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.63500000000000001,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.32600000000000001}]},{"osd":4,"last update":"Fri Apr 12 08:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58599999999999997,"15min":0.58799999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59999999999999998,"15min":0.60699999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.59499999999999997}]},{"osd":5,"last update":"Fri Apr 12 08:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.59899999999999998,"15min":0.60999999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.72899999999999998,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.61799999999999999,"15min":0.624},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.70299999999999996}]},{"osd":6,"last update":"Fri Apr 12 08:43:17 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.61499999999999999,"15min":0.621},"min":{"1min":0.38400000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.76700000000000002,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.623,"15min":0.629},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.81100000000000005,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.65100000000000002}]},{"osd":7,"last update":"Fri Apr 12 08:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.63400000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411375,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:44 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.36499999999999999,"15min":0.35899999999999999},"min":{"1min":0.28199999999999997,"5min":0.218,"15min":0.218},"max":{"1min":0.48999999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.439},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.39500000000000002,"15min":0.38900000000000001},"min":{"1min":0.252,"5min":0.248,"15min":0.248},"max":{"1min":0.49299999999999999,"5min":0.629,"15min":0.629},"last":0.55700000000000005}]},{"osd":1,"last update":"Fri Apr 12 08:42:44 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.38,"15min":0.371},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.56299999999999994,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41099999999999998,"15min":0.39400000000000002},"min":{"1min":0.34499999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.53300000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.52000000000000002}]},{"osd":3,"last update":"Fri Apr 12 08:43:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.46999999999999997,"15min":0.48399999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.58099999999999996,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.45700000000000002,"15min":0.47099999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57199999999999995,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.48699999999999999}]},{"osd":4,"last update":"Fri Apr 12 08:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":1.2210000000000001,"15min":1.335},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.82099999999999995,"5min":14.991,"15min":14.991},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.59399999999999997,"15min":0.59799999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.74399999999999999,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.625}]},{"osd":5,"last update":"Fri Apr 12 08:42:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.65200000000000002,"15min":0.66300000000000003},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.78300000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.63500000000000001,"15min":0.64500000000000002},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.68100000000000005}]},{"osd":6,"last update":"Fri Apr 12 08:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.627,"15min":0.629},"min":{"1min":0.42899999999999999,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83999999999999997,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.64200000000000002,"15min":0.64600000000000002},"min":{"1min":0.45100000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.79800000000000004}]},{"osd":7,"last update":"Fri Apr 12 08:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.69899999999999995}]}]},{"osd":3,"up_from":26,"seq":111669149738,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:43:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.38,"15min":0.38900000000000001},"min":{"1min":0.23999999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.50800000000000001,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.41599999999999998,"15min":0.432},"min":{"1min":0.26300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.439,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.46300000000000002}]},{"osd":1,"last update":"Fri Apr 12 08:43:09 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.436,"15min":0.45600000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.503,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.375,"5min":0.44600000000000001,"15min":0.46999999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.47199999999999998,"5min":0.751,"15min":0.751},"last":0.45100000000000001}]},{"osd":2,"last update":"Fri Apr 12 08:43:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.46800000000000003,"15min":0.48999999999999999},"min":{"1min":0.317,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.52900000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.437,"15min":0.45300000000000001},"min":{"1min":0.26700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.55800000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.41899999999999998}]},{"osd":4,"last update":"Fri Apr 12 08:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.59099999999999997,"15min":0.59699999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.59199999999999997,"15min":0.59999999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71599999999999997,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.67500000000000004}]},{"osd":5,"last update":"Fri Apr 12 08:42:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.59699999999999998,"15min":0.60199999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.60199999999999998,"15min":0.60999999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.76200000000000001,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.65500000000000003}]},{"osd":6,"last update":"Fri Apr 12 08:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42899999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.57499999999999996,"15min":0.56999999999999995},"min":{"1min":0.46400000000000002,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.628}]},{"osd":7,"last update":"Fri Apr 12 08:42:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.70699999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986213,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.42399999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72899999999999998,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.52800000000000002}]},{"osd":1,"last update":"Fri Apr 12 08:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.41399999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.69999999999999996,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.68999999999999995}]},{"osd":2,"last update":"Fri Apr 12 08:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.54500000000000004,"15min":0.54200000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.629},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79800000000000004,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.498}]},{"osd":3,"last update":"Fri Apr 12 08:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39700000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76800000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.65000000000000002}]},{"osd":5,"last update":"Fri Apr 12 08:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.38300000000000001,"15min":0.377},"min":{"1min":0.30499999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.42699999999999999,"15min":0.42099999999999999},"min":{"1min":0.33500000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.73899999999999999,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.39500000000000002}]},{"osd":6,"last update":"Fri Apr 12 08:43:12 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.41499999999999998,"15min":0.39900000000000002},"min":{"1min":0.35699999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.70899999999999996,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.5,"15min":0.48599999999999999},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.81699999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.47299999999999998}]},{"osd":7,"last update":"Fri Apr 12 08:42:35 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.50900000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724576,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.52600000000000002,"15min":0.53100000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.52800000000000002,"15min":0.52600000000000002},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.68600000000000005}]},{"osd":1,"last update":"Fri Apr 12 08:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.52100000000000002,"15min":0.52600000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.61499999999999999}]},{"osd":2,"last update":"Fri Apr 12 08:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.52900000000000003,"15min":0.53200000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.55100000000000005,"15min":0.55700000000000005},"min":{"1min":0.39900000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.59999999999999998}]},{"osd":3,"last update":"Fri Apr 12 08:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52000000000000002,"15min":0.52200000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.627},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56399999999999995,"15min":0.56499999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.71099999999999997}]},{"osd":4,"last update":"Fri Apr 12 08:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39700000000000002,"15min":0.39200000000000002},"min":{"1min":0.312,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.40799999999999997,"15min":0.40000000000000002},"min":{"1min":0.29799999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.46200000000000002}]},{"osd":6,"last update":"Fri Apr 12 08:43:10 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.439,"15min":0.42899999999999999},"min":{"1min":0.373,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.46899999999999997,"15min":0.45400000000000001},"min":{"1min":0.33500000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.505}]},{"osd":7,"last update":"Fri Apr 12 08:42:36 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.56699999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T08:43:21.185 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-12T08:43:21.185 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-12T08:43:21.185 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-12T08:43:21.185 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T08:43:21.194 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:21 smithi114 ceph-mon[33573]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:43:21.194 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:21.195 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:21.443 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:22.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:22 smithi090 ceph-mon[29100]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T08:43:22.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:22 smithi090 ceph-mon[29100]: Reconfiguring daemon prometheus.a on smithi114 2024-04-12T08:43:22.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:22 smithi090 ceph-mon[29100]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:22.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:22 smithi090 ceph-mon[40003]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T08:43:22.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:22 smithi090 ceph-mon[40003]: Reconfiguring daemon prometheus.a on smithi114 2024-04-12T08:43:22.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:22 smithi090 ceph-mon[40003]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:22.299 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:22 smithi114 ceph-mon[33573]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T08:43:22.300 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:22 smithi114 ceph-mon[33573]: Reconfiguring daemon prometheus.a on smithi114 2024-04-12T08:43:22.300 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:22 smithi114 ceph-mon[33573]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:22.533 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:22 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:43:22.395Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000242495s 2024-04-12T08:43:23.289 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:43:23.290 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-12T08:43:23.617 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 systemd[1]: Stopping Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:43:23.994 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-12T08:43:23.994 INFO:tasks.cephadm:Setup complete, yielding 2024-04-12T08:43:23.994 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T08:43:24.004 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi090.front.sepia.ceph.com 2024-04-12T08:43:24.005 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-04-12T08:43:24.028 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.727Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-12T08:43:24.028 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.727Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-12T08:43:24.028 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.727Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-12T08:43:24.028 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.727Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-12T08:43:24.028 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.727Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-12T08:43:24.028 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.727Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-12T08:43:24.028 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.727Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-12T08:43:24.029 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.727Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-12T08:43:24.029 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.727Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-12T08:43:24.029 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.728Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-12T08:43:24.029 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.728Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-12T08:43:24.029 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[60343]: ts=2024-04-12T08:43:23.728Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-12T08:43:24.029 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:23 smithi114 podman[63598]: 2024-04-12 08:43:23.737607184 +0000 UTC m=+0.135288339 container died 1d96e879a67c45a02bdb88b713c8087c0cf434ee911df327e6c39c562b3d0b87 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:24.257 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:24 smithi114 ceph-mon[33573]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:24 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/589960540' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T08:43:24.367 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 podman[63598]: 2024-04-12 08:43:24.135585016 +0000 UTC m=+0.533266167 container cleanup 1d96e879a67c45a02bdb88b713c8087c0cf434ee911df327e6c39c562b3d0b87 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:24.367 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 bash[63598]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a 2024-04-12T08:43:24.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:24 smithi090 ceph-mon[29100]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:24.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:24 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/589960540' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T08:43:24.380 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:24 smithi090 ceph-mon[40003]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:24.380 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:24 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/589960540' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T08:43:24.769 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 podman[63612]: 2024-04-12 08:43:24.403542559 +0000 UTC m=+0.665026560 container remove 1d96e879a67c45a02bdb88b713c8087c0cf434ee911df327e6c39c562b3d0b87 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:24.769 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-12T08:43:24.769 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 systemd[1]: Stopped Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:43:24.769 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@prometheus.a.service: Consumed 1.028s CPU time. 2024-04-12T08:43:24.769 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 systemd[1]: Starting Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:43:25.019 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 podman[63709]: 2024-04-12 08:43:24.671374334 +0000 UTC m=+0.018716451 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-12T08:43:25.020 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 podman[63709]: 2024-04-12 08:43:24.771265343 +0000 UTC m=+0.118607502 container create 94ac082be83bd7d49452d206a5b86ed9ab3584efb8b6c8acf6270c0147e544d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:25.020 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 podman[63709]: 2024-04-12 08:43:24.973331062 +0000 UTC m=+0.320673178 container init 94ac082be83bd7d49452d206a5b86ed9ab3584efb8b6c8acf6270c0147e544d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:25.020 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 podman[63709]: 2024-04-12 08:43:24.975849025 +0000 UTC m=+0.323191143 container start 94ac082be83bd7d49452d206a5b86ed9ab3584efb8b6c8acf6270c0147e544d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:43:25.020 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:24.999Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-12T08:43:25.020 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:24.999Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-12T08:43:25.020 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:24.999Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi114 (none))" 2024-04-12T08:43:25.020 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:24.999Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-12T08:43:25.020 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:24.999Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-12T08:43:25.020 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.000Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-12T08:43:25.020 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.001Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.001Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.001Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.004Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.004Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.403µs 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.004Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.004Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.005Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.005Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.908µs wal_replay_duration=626.561µs wbl_replay_duration=95ns total_replay_duration=686.6µs 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.005Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.005Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-12T08:43:25.021 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.005Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-12T08:43:25.022 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.019Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.449943ms db_storage=670ns remote_storage=1.121µs web_handler=364ns query_engine=503ns scrape=203.045µs scrape_sd=31.641µs notify=17.132µs notify_sd=5.897µs rules=12.980233ms tracing=5.438µs 2024-04-12T08:43:25.022 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.019Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-12T08:43:25.022 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:43:25.019Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-12T08:43:25.291 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 bash[63709]: 94ac082be83bd7d49452d206a5b86ed9ab3584efb8b6c8acf6270c0147e544d1 2024-04-12T08:43:25.291 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:43:25 smithi114 systemd[1]: Started Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:43:25.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STOPPING 2024-04-12T08:43:25.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T08:43:25.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STOPPED 2024-04-12T08:43:25.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STARTING 2024-04-12T08:43:25.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Serving on http://:::9283 2024-04-12T08:43:25.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STARTED 2024-04-12T08:43:25.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STOPPING 2024-04-12T08:43:25.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T08:43:25.424 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STOPPED 2024-04-12T08:43:25.424 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STARTING 2024-04-12T08:43:25.424 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Serving on http://:::9283 2024-04-12T08:43:25.424 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STARTED 2024-04-12T08:43:25.424 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STOPPING 2024-04-12T08:43:25.424 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T08:43:25.424 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STOPPED 2024-04-12T08:43:25.424 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STARTING 2024-04-12T08:43:25.809 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Serving on http://:::9283 2024-04-12T08:43:25.809 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:43:25] ENGINE Bus STARTED 2024-04-12T08:43:26.103 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:26.103 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.103 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.103 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:43:26.104 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi090.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T08:43:26.104 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.104 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:43:26.104 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi114.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T08:43:26.104 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.104 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:43:26.104 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi114.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T08:43:26.104 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.104 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:43:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi090.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi114.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi114.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:26 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi090.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi114.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi114.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T08:43:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:26.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:26 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:43:27.366 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:27 smithi114 ceph-mon[33573]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:43:27.366 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:27 smithi114 ceph-mon[33573]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi090.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T08:43:27.366 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:27 smithi114 ceph-mon[33573]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:43:27.366 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:27 smithi114 ceph-mon[33573]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi114.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T08:43:27.366 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:27 smithi114 ceph-mon[33573]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:43:27.366 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:27 smithi114 ceph-mon[33573]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi114.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T08:43:27.366 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:27 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-12T08:43:27.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[40003]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[40003]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi090.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[40003]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[40003]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi114.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[40003]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[40003]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi114.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[29100]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[29100]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi090.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[29100]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[29100]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi114.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T08:43:27.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[29100]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:43:27.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[29100]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi114.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T08:43:27.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:27 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-12T08:43:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:28 smithi114 ceph-mon[33573]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:43:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:28 smithi114 ceph-mon[33573]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:28 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-12T08:43:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:28 smithi114 ceph-mon[33573]: osdmap e65: 8 total, 8 up, 8 in 2024-04-12T08:43:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:28 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-12T08:43:28.393 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-mon[40003]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:43:28.393 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-mon[40003]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:28.393 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-12T08:43:28.393 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-mon[40003]: osdmap e65: 8 total, 8 up, 8 in 2024-04-12T08:43:28.393 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-12T08:43:28.393 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-mon[29100]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:43:28.394 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-mon[29100]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:28.394 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-12T08:43:28.394 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-mon[29100]: osdmap e65: 8 total, 8 up, 8 in 2024-04-12T08:43:28.394 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-12T08:43:28.394 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:28 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a[29096]: 2024-04-12T08:43:28.135+0000 7fdb690e8700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T08:43:29.089 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-12T08:43:29.377 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-12T08:43:29.377 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: osdmap e66: 8 total, 8 up, 8 in 2024-04-12T08:43:29.377 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-12T08:43:29.377 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T08:43:29.377 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: osdmap e67: 8 total, 8 up, 8 in 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: fsmap foo:0 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T08:43:29.378 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: osdmap e66: 8 total, 8 up, 8 in 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: osdmap e67: 8 total, 8 up, 8 in 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: fsmap foo:0 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.379 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.380 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.380 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:29.380 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:29.380 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.380 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:43:29.380 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T08:43:29.380 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:29 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:29.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-12T08:43:29.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: osdmap e66: 8 total, 8 up, 8 in 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: osdmap e67: 8 total, 8 up, 8 in 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: fsmap foo:0 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:29.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:29.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:29.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:43:29.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T08:43:29.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:29 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:30.398 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:30 smithi090 ceph-mon[40003]: Saving service mds.foo spec with placement count:2 2024-04-12T08:43:30.398 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:30 smithi090 ceph-mon[40003]: pgmap v53: 73 pgs: 13 creating+peering, 6 creating+activating, 54 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:30.398 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:30 smithi090 ceph-mon[40003]: Deploying daemon mds.foo.smithi090.rxdrqj on smithi090 2024-04-12T08:43:30.399 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:30 smithi090 ceph-mon[40003]: osdmap e68: 8 total, 8 up, 8 in 2024-04-12T08:43:30.399 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:30 smithi090 ceph-mon[29100]: Saving service mds.foo spec with placement count:2 2024-04-12T08:43:30.399 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:30 smithi090 ceph-mon[29100]: pgmap v53: 73 pgs: 13 creating+peering, 6 creating+activating, 54 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:30.399 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:30 smithi090 ceph-mon[29100]: Deploying daemon mds.foo.smithi090.rxdrqj on smithi090 2024-04-12T08:43:30.399 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:30 smithi090 ceph-mon[29100]: osdmap e68: 8 total, 8 up, 8 in 2024-04-12T08:43:30.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:30 smithi114 ceph-mon[33573]: Saving service mds.foo spec with placement count:2 2024-04-12T08:43:30.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:30 smithi114 ceph-mon[33573]: pgmap v53: 73 pgs: 13 creating+peering, 6 creating+activating, 54 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:30.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:30 smithi114 ceph-mon[33573]: Deploying daemon mds.foo.smithi090.rxdrqj on smithi090 2024-04-12T08:43:30.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:30 smithi114 ceph-mon[33573]: osdmap e68: 8 total, 8 up, 8 in 2024-04-12T08:43:30.627 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:30.672 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:43:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:43:30.397Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002781797s 2024-04-12T08:43:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:31 smithi090 ceph-mon[40003]: osdmap e69: 8 total, 8 up, 8 in 2024-04-12T08:43:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:31 smithi090 ceph-mon[40003]: pgmap v56: 73 pgs: 13 creating+peering, 6 creating+activating, 54 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:31 smithi090 ceph-mon[29100]: osdmap e69: 8 total, 8 up, 8 in 2024-04-12T08:43:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:31 smithi090 ceph-mon[29100]: pgmap v56: 73 pgs: 13 creating+peering, 6 creating+activating, 54 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:31 smithi114 ceph-mon[33573]: osdmap e69: 8 total, 8 up, 8 in 2024-04-12T08:43:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:31 smithi114 ceph-mon[33573]: pgmap v56: 73 pgs: 13 creating+peering, 6 creating+activating, 54 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:32.069 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:31 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:43:31] "GET /metrics HTTP/1.1" 200 203244 "" "Prometheus/2.43.0" 2024-04-12T08:43:33.197 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-12T08:43:33.540 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:33.775 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[40003]: pgmap v57: 73 pgs: 13 creating+peering, 6 creating+activating, 54 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:33.775 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.775 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:33.775 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.775 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.775 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[29100]: pgmap v57: 73 pgs: 13 creating+peering, 6 creating+activating, 54 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:43:33.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T08:43:33.777 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:33 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:33.962 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:33 smithi114 ceph-mon[33573]: pgmap v57: 73 pgs: 13 creating+peering, 6 creating+activating, 54 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:33.962 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:33 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.962 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:33 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:33.962 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:33 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.962 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:33 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.962 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:33 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:33.962 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:33 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:43:33.962 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:33 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T08:43:33.963 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:33 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:34.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[40003]: Deploying daemon mds.foo.smithi114.nbzjob on smithi114 2024-04-12T08:43:34.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[40003]: mds.? [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] up:boot 2024-04-12T08:43:34.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[40003]: daemon mds.foo.smithi090.rxdrqj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-12T08:43:34.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[40003]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T08:43:34.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[40003]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[40003]: Cluster is now healthy 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[40003]: fsmap foo:0 1 up:standby 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[29100]: Deploying daemon mds.foo.smithi114.nbzjob on smithi114 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[29100]: mds.? [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] up:boot 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[29100]: daemon mds.foo.smithi090.rxdrqj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[29100]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[29100]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[29100]: Cluster is now healthy 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[29100]: fsmap foo:0 1 up:standby 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[29100]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:creating} 2024-04-12T08:43:34.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[29100]: daemon mds.foo.smithi090.rxdrqj is now active in filesystem foo as rank 0 2024-04-12T08:43:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:34 smithi114 ceph-mon[33573]: Deploying daemon mds.foo.smithi114.nbzjob on smithi114 2024-04-12T08:43:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:34 smithi114 ceph-mon[33573]: mds.? [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] up:boot 2024-04-12T08:43:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:34 smithi114 ceph-mon[33573]: daemon mds.foo.smithi090.rxdrqj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-12T08:43:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:34 smithi114 ceph-mon[33573]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T08:43:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:34 smithi114 ceph-mon[33573]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-12T08:43:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:34 smithi114 ceph-mon[33573]: Cluster is now healthy 2024-04-12T08:43:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:34 smithi114 ceph-mon[33573]: fsmap foo:0 1 up:standby 2024-04-12T08:43:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:34 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:43:34.938 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:34 smithi114 ceph-mon[33573]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:creating} 2024-04-12T08:43:34.939 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:34 smithi114 ceph-mon[33573]: daemon mds.foo.smithi090.rxdrqj is now active in filesystem foo as rank 0 2024-04-12T08:43:34.959 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:43:34.959 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[40003]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:creating} 2024-04-12T08:43:34.959 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:34 smithi090 ceph-mon[40003]: daemon mds.foo.smithi090.rxdrqj is now active in filesystem foo as rank 0 2024-04-12T08:43:35.320 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:43:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:43:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:43:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:35 smithi114 ceph-mon[33573]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:35 smithi114 ceph-mon[33573]: mds.? [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] up:active 2024-04-12T08:43:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:35 smithi114 ceph-mon[33573]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 2024-04-12T08:43:35.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:35 smithi090 ceph-mon[40003]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:35.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:35 smithi090 ceph-mon[40003]: mds.? [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] up:active 2024-04-12T08:43:35.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:35 smithi090 ceph-mon[40003]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 2024-04-12T08:43:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:35 smithi090 ceph-mon[29100]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:35 smithi090 ceph-mon[29100]: mds.? [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] up:active 2024-04-12T08:43:35.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:35 smithi090 ceph-mon[29100]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 2024-04-12T08:43:36.356 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-12T08:43:36.365 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-12T08:43:36.365 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-12T08:43:36.365 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph mgr dump --format=json 2024-04-12T08:43:37.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:43:37.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[29100]: mds.? [v2:172.21.15.114:6832/562045409,v1:172.21.15.114:6833/562045409] up:boot 2024-04-12T08:43:37.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[29100]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:43:37.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:43:37.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:43:37.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[40003]: mds.? [v2:172.21.15.114:6832/562045409,v1:172.21.15.114:6833/562045409] up:boot 2024-04-12T08:43:37.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[40003]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:43:37.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:36 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:43:37.180 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:37.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:36 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:36 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:36 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:36 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:36 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:37.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:36 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:43:37.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:36 smithi114 ceph-mon[33573]: mds.? [v2:172.21.15.114:6832/562045409,v1:172.21.15.114:6833/562045409] up:boot 2024-04-12T08:43:37.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:36 smithi114 ceph-mon[33573]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:43:37.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:36 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:43:38.207 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:37 smithi114 ceph-mon[33573]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:38.207 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:37 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:38.304 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:37 smithi090 ceph-mon[40003]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:38.304 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:37 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:38.305 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:37 smithi090 ceph-mon[29100]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:38.305 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:37 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:39.117 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:43:39.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:38 smithi114 ceph-mon[33573]: mds.? [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] up:active 2024-04-12T08:43:39.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:38 smithi114 ceph-mon[33573]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:43:39.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:38 smithi090 ceph-mon[29100]: mds.? [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] up:active 2024-04-12T08:43:39.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:38 smithi090 ceph-mon[29100]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:43:39.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:38 smithi090 ceph-mon[40003]: mds.? [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] up:active 2024-04-12T08:43:39.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:38 smithi090 ceph-mon[40003]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:43:39.872 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":20,"active_gid":14421,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6800","nonce":3500986148},{"type":"v1","addr":"172.21.15.90:6801","nonce":3500986148}]},"active_addr":"172.21.15.90:6801/3500986148","active_change":"2024-04-12T08:42:02.589309+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24328,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.90:8443/","prometheus":"http://172.21.15.90:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":1763471199}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":794297450}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":3912205664}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":4275803179}]}]} 2024-04-12T08:43:39.876 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-12T08:43:39.876 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-12T08:43:39.877 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T08:43:40.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:39 smithi090 ceph-mon[40003]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-04-12T08:43:40.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:39 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1487811990' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T08:43:40.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:39 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:40.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:39 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:40.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:39 smithi090 ceph-mon[29100]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-04-12T08:43:40.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:39 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1487811990' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T08:43:40.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:39 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:40.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:39 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:40.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:39 smithi114 ceph-mon[33573]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-04-12T08:43:40.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:39 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1487811990' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T08:43:40.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:39 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:40.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:39 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:40.375 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:41.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[40003]: mds.? [v2:172.21.15.114:6832/562045409,v1:172.21.15.114:6833/562045409] up:standby 2024-04-12T08:43:41.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[40003]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:43:41.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[40003]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-12T08:43:41.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:41.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:41.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:41.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:41.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:41.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:43:41.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[29100]: mds.? [v2:172.21.15.114:6832/562045409,v1:172.21.15.114:6833/562045409] up:standby 2024-04-12T08:43:41.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[29100]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:43:41.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[29100]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-12T08:43:41.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:41.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:41.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:41.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:41.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:41.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:43:41.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:41 smithi114 ceph-mon[33573]: mds.? [v2:172.21.15.114:6832/562045409,v1:172.21.15.114:6833/562045409] up:standby 2024-04-12T08:43:41.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:41 smithi114 ceph-mon[33573]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:43:41.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:41 smithi114 ceph-mon[33573]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-12T08:43:41.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:41 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:41.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:41 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:41.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:41 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:41.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:41 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:41.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:41 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:41.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:41 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:43:42.165 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:43:41] "GET /metrics HTTP/1.1" 200 208138 "" "Prometheus/2.43.0" 2024-04-12T08:43:43.565 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:43:43.565 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":69,"fsid":"96735376-f8a7-11ee-bc8c-c7b262605968","created":"2024-04-12T08:36:35.370883+0000","modified":"2024-04-12T08:43:30.138898+0000","last_up_change":"2024-04-12T08:41:33.609734+0000","last_in_change":"2024-04-12T08:41:16.832300+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T08:39:38.877058+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T08:42:17.012082+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T08:43:26.507787+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T08:43:27.356729+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"3e888cb8-b0cb-4cb8-95ae-7dd29d090968","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6803","nonce":937815701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6805","nonce":937815701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6809","nonce":937815701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6807","nonce":937815701}]},"public_addr":"172.21.15.90:6803/937815701","cluster_addr":"172.21.15.90:6805/937815701","heartbeat_back_addr":"172.21.15.90:6809/937815701","heartbeat_front_addr":"172.21.15.90:6807/937815701","state":["exists","up"]},{"osd":1,"uuid":"637213e4-7019-4430-ab01-b7196988f82c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6811","nonce":2625409332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6813","nonce":2625409332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6817","nonce":2625409332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6815","nonce":2625409332}]},"public_addr":"172.21.15.90:6811/2625409332","cluster_addr":"172.21.15.90:6813/2625409332","heartbeat_back_addr":"172.21.15.90:6817/2625409332","heartbeat_front_addr":"172.21.15.90:6815/2625409332","state":["exists","up"]},{"osd":2,"uuid":"99513c85-6468-4377-85b8-3dc52f0d0f2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6819","nonce":3723833287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6821","nonce":3723833287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6825","nonce":3723833287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6823","nonce":3723833287}]},"public_addr":"172.21.15.90:6819/3723833287","cluster_addr":"172.21.15.90:6821/3723833287","heartbeat_back_addr":"172.21.15.90:6825/3723833287","heartbeat_front_addr":"172.21.15.90:6823/3723833287","state":["exists","up"]},{"osd":3,"uuid":"afd01784-8987-4954-9be5-57d950beb611","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6827","nonce":1425929386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6829","nonce":1425929386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6833","nonce":1425929386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6831","nonce":1425929386}]},"public_addr":"172.21.15.90:6827/1425929386","cluster_addr":"172.21.15.90:6829/1425929386","heartbeat_back_addr":"172.21.15.90:6833/1425929386","heartbeat_front_addr":"172.21.15.90:6831/1425929386","state":["exists","up"]},{"osd":4,"uuid":"596f9e87-d99e-423c-902f-f14c5dcdcf36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6801","nonce":1802592095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6803","nonce":1802592095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6807","nonce":1802592095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6805","nonce":1802592095}]},"public_addr":"172.21.15.114:6801/1802592095","cluster_addr":"172.21.15.114:6803/1802592095","heartbeat_back_addr":"172.21.15.114:6807/1802592095","heartbeat_front_addr":"172.21.15.114:6805/1802592095","state":["exists","up"]},{"osd":5,"uuid":"b2374027-89a9-4670-9593-3ddf9393b286","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6809","nonce":117630080}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6811","nonce":117630080}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6815","nonce":117630080}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6813","nonce":117630080}]},"public_addr":"172.21.15.114:6809/117630080","cluster_addr":"172.21.15.114:6811/117630080","heartbeat_back_addr":"172.21.15.114:6815/117630080","heartbeat_front_addr":"172.21.15.114:6813/117630080","state":["exists","up"]},{"osd":6,"uuid":"487cdd03-1123-4386-9237-97b2bc6dd92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6817","nonce":2139728372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6819","nonce":2139728372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6823","nonce":2139728372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6821","nonce":2139728372}]},"public_addr":"172.21.15.114:6817/2139728372","cluster_addr":"172.21.15.114:6819/2139728372","heartbeat_back_addr":"172.21.15.114:6823/2139728372","heartbeat_front_addr":"172.21.15.114:6821/2139728372","state":["exists","up"]},{"osd":7,"uuid":"0d46acbb-c271-4f1f-b5ba-090a44443308","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6825","nonce":3563377268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6827","nonce":3563377268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6831","nonce":3563377268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6829","nonce":3563377268}]},"public_addr":"172.21.15.114:6825/3563377268","cluster_addr":"172.21.15.114:6827/3563377268","heartbeat_back_addr":"172.21.15.114:6831/3563377268","heartbeat_front_addr":"172.21.15.114:6829/3563377268","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:38:46.298090+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:11.083177+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:35.186704+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:59.618196+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:21.910327+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:44.541214+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:08.064023+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:31.949563+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/3964952044":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6800/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/3166816614":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/1996176058":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6801/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3357701831":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3631859869":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6800/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3110959800":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/806687008":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6800/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3408352967":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/577805764":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/721288352":"2024-04-13T08:37:02.805251+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T08:43:43.853 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:43 smithi090 ceph-mon[40003]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:43.854 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:43 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3688518065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:43:43.854 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:43 smithi090 ceph-mon[29100]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:43.854 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:43 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3688518065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:43:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:43 smithi114 ceph-mon[33573]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:43 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3688518065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:43:44.362 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-12T08:43:44.363 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T08:43:45.075 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:45.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:43:45 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:43:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:43:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:45 smithi090 ceph-mon[29100]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:45 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:45 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:45 smithi090 ceph-mon[40003]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:45 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:45 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:45 smithi114 ceph-mon[33573]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:45 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:45 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:47.686 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:43:47.687 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":69,"fsid":"96735376-f8a7-11ee-bc8c-c7b262605968","created":"2024-04-12T08:36:35.370883+0000","modified":"2024-04-12T08:43:30.138898+0000","last_up_change":"2024-04-12T08:41:33.609734+0000","last_in_change":"2024-04-12T08:41:16.832300+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T08:39:38.877058+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T08:42:17.012082+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T08:43:26.507787+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T08:43:27.356729+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"3e888cb8-b0cb-4cb8-95ae-7dd29d090968","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6803","nonce":937815701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6805","nonce":937815701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6809","nonce":937815701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":937815701},{"type":"v1","addr":"172.21.15.90:6807","nonce":937815701}]},"public_addr":"172.21.15.90:6803/937815701","cluster_addr":"172.21.15.90:6805/937815701","heartbeat_back_addr":"172.21.15.90:6809/937815701","heartbeat_front_addr":"172.21.15.90:6807/937815701","state":["exists","up"]},{"osd":1,"uuid":"637213e4-7019-4430-ab01-b7196988f82c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6811","nonce":2625409332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6813","nonce":2625409332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6817","nonce":2625409332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":2625409332},{"type":"v1","addr":"172.21.15.90:6815","nonce":2625409332}]},"public_addr":"172.21.15.90:6811/2625409332","cluster_addr":"172.21.15.90:6813/2625409332","heartbeat_back_addr":"172.21.15.90:6817/2625409332","heartbeat_front_addr":"172.21.15.90:6815/2625409332","state":["exists","up"]},{"osd":2,"uuid":"99513c85-6468-4377-85b8-3dc52f0d0f2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6819","nonce":3723833287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6821","nonce":3723833287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6825","nonce":3723833287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":3723833287},{"type":"v1","addr":"172.21.15.90:6823","nonce":3723833287}]},"public_addr":"172.21.15.90:6819/3723833287","cluster_addr":"172.21.15.90:6821/3723833287","heartbeat_back_addr":"172.21.15.90:6825/3723833287","heartbeat_front_addr":"172.21.15.90:6823/3723833287","state":["exists","up"]},{"osd":3,"uuid":"afd01784-8987-4954-9be5-57d950beb611","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6827","nonce":1425929386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6829","nonce":1425929386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6833","nonce":1425929386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":1425929386},{"type":"v1","addr":"172.21.15.90:6831","nonce":1425929386}]},"public_addr":"172.21.15.90:6827/1425929386","cluster_addr":"172.21.15.90:6829/1425929386","heartbeat_back_addr":"172.21.15.90:6833/1425929386","heartbeat_front_addr":"172.21.15.90:6831/1425929386","state":["exists","up"]},{"osd":4,"uuid":"596f9e87-d99e-423c-902f-f14c5dcdcf36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6801","nonce":1802592095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6803","nonce":1802592095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6807","nonce":1802592095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6805","nonce":1802592095}]},"public_addr":"172.21.15.114:6801/1802592095","cluster_addr":"172.21.15.114:6803/1802592095","heartbeat_back_addr":"172.21.15.114:6807/1802592095","heartbeat_front_addr":"172.21.15.114:6805/1802592095","state":["exists","up"]},{"osd":5,"uuid":"b2374027-89a9-4670-9593-3ddf9393b286","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6809","nonce":117630080}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6811","nonce":117630080}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6815","nonce":117630080}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6813","nonce":117630080}]},"public_addr":"172.21.15.114:6809/117630080","cluster_addr":"172.21.15.114:6811/117630080","heartbeat_back_addr":"172.21.15.114:6815/117630080","heartbeat_front_addr":"172.21.15.114:6813/117630080","state":["exists","up"]},{"osd":6,"uuid":"487cdd03-1123-4386-9237-97b2bc6dd92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6817","nonce":2139728372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6819","nonce":2139728372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6823","nonce":2139728372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6821","nonce":2139728372}]},"public_addr":"172.21.15.114:6817/2139728372","cluster_addr":"172.21.15.114:6819/2139728372","heartbeat_back_addr":"172.21.15.114:6823/2139728372","heartbeat_front_addr":"172.21.15.114:6821/2139728372","state":["exists","up"]},{"osd":7,"uuid":"0d46acbb-c271-4f1f-b5ba-090a44443308","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6825","nonce":3563377268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6827","nonce":3563377268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6831","nonce":3563377268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6829","nonce":3563377268}]},"public_addr":"172.21.15.114:6825/3563377268","cluster_addr":"172.21.15.114:6827/3563377268","heartbeat_back_addr":"172.21.15.114:6831/3563377268","heartbeat_front_addr":"172.21.15.114:6829/3563377268","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:38:46.298090+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:11.083177+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:35.186704+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:59.618196+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:21.910327+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:44.541214+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:08.064023+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:31.949563+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/3964952044":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6800/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/3166816614":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/1996176058":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6801/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3357701831":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3631859869":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6800/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/3110959800":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/806687008":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6800/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3408352967":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/577805764":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/721288352":"2024-04-13T08:37:02.805251+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T08:43:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:47 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:47 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:47 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:47 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:47 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:47 smithi114 ceph-mon[33573]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:47.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[29100]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:43:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:43:47.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:47.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:47 smithi090 ceph-mon[40003]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:48.505 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-12T08:43:48.506 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-12T08:43:48.506 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-12T08:43:48.507 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-12T08:43:48.507 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-12T08:43:48.507 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-12T08:43:48.508 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-12T08:43:48.508 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-12T08:43:48.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:48 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:48.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:48 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:48.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:48 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4264990467' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:43:48.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:48 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:48 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:48 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:48 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4264990467' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:43:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:48 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:49.058 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:49.059 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:49.059 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:49.059 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:49.061 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:49.067 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:49.067 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:49.071 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:43:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:48 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:48 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:43:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:48 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4264990467' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T08:43:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:48 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:43:49.822 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:49 smithi090 ceph-mon[40003]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:49.822 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:49 smithi090 ceph-mon[29100]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:49 smithi114 ceph-mon[33573]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T08:43:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:51 smithi090 ceph-mon[40003]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:51 smithi090 ceph-mon[29100]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:51.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:43:51 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:43:51] "GET /metrics HTTP/1.1" 200 239252 "" "Prometheus/2.43.0" 2024-04-12T08:43:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:51 smithi114 ceph-mon[33573]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:53 smithi114 ceph-mon[33573]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:54.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:53 smithi090 ceph-mon[29100]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:53 smithi090 ceph-mon[40003]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:55.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:43:55 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:43:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:43:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:55 smithi114 ceph-mon[33573]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:56.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:55 smithi090 ceph-mon[40003]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:55 smithi090 ceph-mon[29100]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:57 smithi090 ceph-mon[29100]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:57.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:57 smithi090 ceph-mon[40003]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:58.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:57 smithi114 ceph-mon[33573]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:59.426 INFO:teuthology.orchestra.run.smithi090.stdout:111669149746 2024-04-12T08:43:59.427 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-12T08:43:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:43:59 smithi090 ceph-mon[40003]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:43:59.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:43:59 smithi090 ceph-mon[29100]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:43:59 smithi114 ceph-mon[33573]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:01.750 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:01 smithi090 ceph-mon[40003]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:01.751 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:01 smithi090 ceph-mon[29100]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:01.752 INFO:teuthology.orchestra.run.smithi090.stdout:133143986221 2024-04-12T08:44:01.753 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-12T08:44:02.069 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:01 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:01] "GET /metrics HTTP/1.1" 200 239252 "" "Prometheus/2.43.0" 2024-04-12T08:44:02.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:01 smithi114 ceph-mon[33573]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:02.459 INFO:teuthology.orchestra.run.smithi090.stdout:167503724585 2024-04-12T08:44:02.459 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-12T08:44:02.973 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2024-04-12T08:44:02.973 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-12T08:44:02.973 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-04-12T08:44:02.973 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:02.973 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2024-04-12T08:44:02.973 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-12T08:44:02.974 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-04-12T08:44:02.974 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2024-04-12T08:44:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-12T08:44:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-04-12T08:44:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:03.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:03 smithi090 ceph-mon[40003]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:03.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2024-04-12T08:44:03.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-12T08:44:03.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-04-12T08:44:03.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:03 smithi090 ceph-mon[40003]: osdmap e70: 8 total, 8 up, 8 in 2024-04-12T08:44:03.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:03 smithi090 ceph-mon[29100]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:03.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2024-04-12T08:44:03.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-12T08:44:03.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-04-12T08:44:03.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:03 smithi090 ceph-mon[29100]: osdmap e70: 8 total, 8 up, 8 in 2024-04-12T08:44:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:03 smithi114 ceph-mon[33573]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2024-04-12T08:44:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-12T08:44:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-04-12T08:44:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:03 smithi114 ceph-mon[33573]: osdmap e70: 8 total, 8 up, 8 in 2024-04-12T08:44:05.012 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:04 smithi114 ceph-mon[33573]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T08:44:05.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:04 smithi090 ceph-mon[40003]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T08:44:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:04 smithi090 ceph-mon[29100]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T08:44:05.321 INFO:teuthology.orchestra.run.smithi090.stdout:34359738434 2024-04-12T08:44:05.322 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-12T08:44:05.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:44:05 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:44:06.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:05 smithi114 ceph-mon[33573]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:06.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:05 smithi090 ceph-mon[29100]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:06.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:05 smithi090 ceph-mon[40003]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:06.338 INFO:teuthology.orchestra.run.smithi090.stdout:55834574910 2024-04-12T08:44:06.338 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-12T08:44:07.684 INFO:teuthology.orchestra.run.smithi090.stdout:236223201312 2024-04-12T08:44:07.684 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-12T08:44:07.800 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:07 smithi090 ceph-mon[29100]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:07.800 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:07 smithi090 ceph-mon[40003]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:07 smithi114 ceph-mon[33573]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:08.274 INFO:teuthology.orchestra.run.smithi090.stdout:201863462949 2024-04-12T08:44:08.274 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-12T08:44:08.536 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:09.106 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:09.108 INFO:teuthology.orchestra.run.smithi090.stdout:77309411385 2024-04-12T08:44:09.108 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-12T08:44:09.112 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:09.112 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:09.112 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:10.064 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:09 smithi090 ceph-mon[29100]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:10.064 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:09 smithi090 ceph-mon[40003]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:10.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:09 smithi114 ceph-mon[33573]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:11.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:11 smithi090 ceph-mon[40003]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:11.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:11 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:11] "GET /metrics HTTP/1.1" 200 239238 "" "Prometheus/2.43.0" 2024-04-12T08:44:11.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:11 smithi090 ceph-mon[29100]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:12.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:11 smithi114 ceph-mon[33573]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:14.019 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:13 smithi090 ceph-mon[40003]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:14.020 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:13 smithi090 ceph-mon[29100]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:14.100 INFO:teuthology.orchestra.run.smithi090.stdout:133143986224 2024-04-12T08:44:14.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:13 smithi114 ceph-mon[33573]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:15.012 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:14 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/374066655' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T08:44:15.071 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:14 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/374066655' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T08:44:15.071 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:14 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/374066655' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T08:44:15.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:44:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:44:15.770 INFO:teuthology.orchestra.run.smithi090.stdout:34359738437 2024-04-12T08:44:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:15 smithi090 ceph-mon[40003]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:15 smithi090 ceph-mon[29100]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:16.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:15 smithi114 ceph-mon[33573]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:16 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1453098926' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T08:44:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:16 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1453098926' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T08:44:17.096 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986221 got 133143986224 for osd.4 2024-04-12T08:44:17.096 DEBUG:teuthology.parallel:result is None 2024-04-12T08:44:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:16 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1453098926' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T08:44:17.811 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738434 got 34359738437 for osd.0 2024-04-12T08:44:17.811 DEBUG:teuthology.parallel:result is None 2024-04-12T08:44:18.028 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:17 smithi090 ceph-mon[40003]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:18.028 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:17 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:18.028 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:17 smithi090 ceph-mon[29100]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:18.028 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:17 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:18.033 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:18.035 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:18.036 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:17 smithi114 ceph-mon[33573]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:17 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:18.253 INFO:teuthology.orchestra.run.smithi090.stdout:111669149750 2024-04-12T08:44:18.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:18 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1578206359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T08:44:18.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:18 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1578206359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T08:44:19.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:18 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1578206359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T08:44:20.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:19 smithi114 ceph-mon[33573]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:20.125 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:19 smithi090 ceph-mon[40003]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:20.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:19 smithi090 ceph-mon[29100]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:20.880 INFO:teuthology.orchestra.run.smithi090.stdout:167503724589 2024-04-12T08:44:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:20 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2656574819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T08:44:21.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:20 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2656574819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T08:44:21.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:20 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2656574819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T08:44:21.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:21 smithi090 ceph-mon[40003]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:21 smithi090 ceph-mon[29100]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:21.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:21] "GET /metrics HTTP/1.1" 200 239256 "" "Prometheus/2.43.0" 2024-04-12T08:44:22.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:21 smithi114 ceph-mon[33573]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:55834574913 2024-04-12T08:44:22.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:22 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1430713033' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T08:44:22.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:22 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1430713033' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T08:44:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:22 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1430713033' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T08:44:23.383 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574910 got 55834574913 for osd.1 2024-04-12T08:44:23.384 DEBUG:teuthology.parallel:result is None 2024-04-12T08:44:23.807 INFO:teuthology.orchestra.run.smithi090.stdout:77309411388 2024-04-12T08:44:23.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:23 smithi090 ceph-mon[40003]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:23 smithi090 ceph-mon[29100]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:24.038 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149746 got 111669149750 for osd.3 2024-04-12T08:44:24.039 DEBUG:teuthology.parallel:result is None 2024-04-12T08:44:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:23 smithi114 ceph-mon[33573]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:24.442 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724585 got 167503724589 for osd.5 2024-04-12T08:44:24.442 DEBUG:teuthology.parallel:result is None 2024-04-12T08:44:25.091 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:24 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/836539862' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T08:44:25.091 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:24 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/836539862' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T08:44:25.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:24 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/836539862' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T08:44:25.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:44:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:44:25.120 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411385 got 77309411388 for osd.2 2024-04-12T08:44:25.120 DEBUG:teuthology.parallel:result is None 2024-04-12T08:44:25.550 INFO:teuthology.orchestra.run.smithi090.stdout:201863462953 2024-04-12T08:44:26.067 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:25 smithi090 ceph-mon[29100]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:26.067 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:25 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/497912551' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T08:44:26.068 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:25 smithi090 ceph-mon[40003]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:26.068 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:25 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/497912551' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T08:44:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:25 smithi114 ceph-mon[33573]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:25 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/497912551' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T08:44:26.883 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462949 got 201863462953 for osd.6 2024-04-12T08:44:26.883 DEBUG:teuthology.parallel:result is None 2024-04-12T08:44:27.256 INFO:teuthology.orchestra.run.smithi090.stdout:236223201317 2024-04-12T08:44:27.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:27 smithi090 ceph-mon[40003]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:27.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:27 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/6547785' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T08:44:27.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:27 smithi090 ceph-mon[29100]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:27.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:27 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/6547785' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T08:44:28.024 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201312 got 236223201317 for osd.7 2024-04-12T08:44:28.025 DEBUG:teuthology.parallel:result is None 2024-04-12T08:44:28.025 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-12T08:44:28.025 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T08:44:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:27 smithi114 ceph-mon[33573]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:27 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/6547785' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T08:44:28.284 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:29 smithi114 ceph-mon[33573]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:30.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:29 smithi090 ceph-mon[40003]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:29 smithi090 ceph-mon[29100]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:30.180 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:44:30.181 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-12T08:44:30.800 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":87,"stamp":"2024-04-12T08:44:28.625762+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386336,"kb_used_data":7472,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574880,"statfs":{"total":767960285184,"available":765516677120,"internally_reserved":0,"allocated":7651328,"data_stored":4307899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002471"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-12T08:44:03.895417+0000","last_change":"2024-04-12T08:44:03.895417+0000","last_active":"2024-04-12T08:44:03.895417+0000","last_peered":"2024-04-12T08:44:03.895417+0000","last_clean":"2024-04-12T08:44:03.895417+0000","last_became_active":"2024-04-12T08:44:03.895040+0000","last_became_peered":"2024-04-12T08:44:03.895040+0000","last_unstale":"2024-04-12T08:44:03.895417+0000","last_undegraded":"2024-04-12T08:44:03.895417+0000","last_fullsized":"2024-04-12T08:44:03.895417+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:23:30.820364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928856+0000","last_change":"2024-04-12T08:43:28.126923+0000","last_active":"2024-04-12T08:44:02.928856+0000","last_peered":"2024-04-12T08:44:02.928856+0000","last_clean":"2024-04-12T08:44:02.928856+0000","last_became_active":"2024-04-12T08:43:28.126605+0000","last_became_peered":"2024-04-12T08:43:28.126605+0000","last_unstale":"2024-04-12T08:44:02.928856+0000","last_undegraded":"2024-04-12T08:44:02.928856+0000","last_fullsized":"2024-04-12T08:44:02.928856+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:53:04.343113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724730+0000","last_change":"2024-04-12T08:43:28.319152+0000","last_active":"2024-04-12T08:44:02.724730+0000","last_peered":"2024-04-12T08:44:02.724730+0000","last_clean":"2024-04-12T08:44:02.724730+0000","last_became_active":"2024-04-12T08:43:28.318772+0000","last_became_peered":"2024-04-12T08:43:28.318772+0000","last_unstale":"2024-04-12T08:44:02.724730+0000","last_undegraded":"2024-04-12T08:44:02.724730+0000","last_fullsized":"2024-04-12T08:44:02.724730+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:11:08.333819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725586+0000","last_change":"2024-04-12T08:43:28.128857+0000","last_active":"2024-04-12T08:44:02.725586+0000","last_peered":"2024-04-12T08:44:02.725586+0000","last_clean":"2024-04-12T08:44:02.725586+0000","last_became_active":"2024-04-12T08:43:28.127170+0000","last_became_peered":"2024-04-12T08:43:28.127170+0000","last_unstale":"2024-04-12T08:44:02.725586+0000","last_undegraded":"2024-04-12T08:44:02.725586+0000","last_fullsized":"2024-04-12T08:44:02.725586+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:11:07.335293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728465+0000","last_change":"2024-04-12T08:43:28.319145+0000","last_active":"2024-04-12T08:44:02.728465+0000","last_peered":"2024-04-12T08:44:02.728465+0000","last_clean":"2024-04-12T08:44:02.728465+0000","last_became_active":"2024-04-12T08:43:28.318896+0000","last_became_peered":"2024-04-12T08:43:28.318896+0000","last_unstale":"2024-04-12T08:44:02.728465+0000","last_undegraded":"2024-04-12T08:44:02.728465+0000","last_fullsized":"2024-04-12T08:44:02.728465+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:04:09.427910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720474+0000","last_change":"2024-04-12T08:43:28.137332+0000","last_active":"2024-04-12T08:44:02.720474+0000","last_peered":"2024-04-12T08:44:02.720474+0000","last_clean":"2024-04-12T08:44:02.720474+0000","last_became_active":"2024-04-12T08:43:28.136749+0000","last_became_peered":"2024-04-12T08:43:28.136749+0000","last_unstale":"2024-04-12T08:44:02.720474+0000","last_undegraded":"2024-04-12T08:44:02.720474+0000","last_fullsized":"2024-04-12T08:44:02.720474+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:20:09.223627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728131+0000","last_change":"2024-04-12T08:43:28.147487+0000","last_active":"2024-04-12T08:44:02.728131+0000","last_peered":"2024-04-12T08:44:02.728131+0000","last_clean":"2024-04-12T08:44:02.728131+0000","last_became_active":"2024-04-12T08:43:28.147265+0000","last_became_peered":"2024-04-12T08:43:28.147265+0000","last_unstale":"2024-04-12T08:44:02.728131+0000","last_undegraded":"2024-04-12T08:44:02.728131+0000","last_fullsized":"2024-04-12T08:44:02.728131+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:40:54.441361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720466+0000","last_change":"2024-04-12T08:43:28.131575+0000","last_active":"2024-04-12T08:44:02.720466+0000","last_peered":"2024-04-12T08:44:02.720466+0000","last_clean":"2024-04-12T08:44:02.720466+0000","last_became_active":"2024-04-12T08:43:28.131277+0000","last_became_peered":"2024-04-12T08:43:28.131277+0000","last_unstale":"2024-04-12T08:44:02.720466+0000","last_undegraded":"2024-04-12T08:44:02.720466+0000","last_fullsized":"2024-04-12T08:44:02.720466+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:22:44.474309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724560+0000","last_change":"2024-04-12T08:43:28.147262+0000","last_active":"2024-04-12T08:44:02.724560+0000","last_peered":"2024-04-12T08:44:02.724560+0000","last_clean":"2024-04-12T08:44:02.724560+0000","last_became_active":"2024-04-12T08:43:28.146959+0000","last_became_peered":"2024-04-12T08:43:28.146959+0000","last_unstale":"2024-04-12T08:44:02.724560+0000","last_undegraded":"2024-04-12T08:44:02.724560+0000","last_fullsized":"2024-04-12T08:44:02.724560+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:04:51.802195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928770+0000","last_change":"2024-04-12T08:43:28.129883+0000","last_active":"2024-04-12T08:44:02.928770+0000","last_peered":"2024-04-12T08:44:02.928770+0000","last_clean":"2024-04-12T08:44:02.928770+0000","last_became_active":"2024-04-12T08:43:28.129633+0000","last_became_peered":"2024-04-12T08:43:28.129633+0000","last_unstale":"2024-04-12T08:44:02.928770+0000","last_undegraded":"2024-04-12T08:44:02.928770+0000","last_fullsized":"2024-04-12T08:44:02.928770+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:13:47.677812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728867+0000","last_change":"2024-04-12T08:43:28.365741+0000","last_active":"2024-04-12T08:44:02.728867+0000","last_peered":"2024-04-12T08:44:02.728867+0000","last_clean":"2024-04-12T08:44:02.728867+0000","last_became_active":"2024-04-12T08:43:28.365425+0000","last_became_peered":"2024-04-12T08:43:28.365425+0000","last_unstale":"2024-04-12T08:44:02.728867+0000","last_undegraded":"2024-04-12T08:44:02.728867+0000","last_fullsized":"2024-04-12T08:44:02.728867+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:13:45.303498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928803+0000","last_change":"2024-04-12T08:43:28.129089+0000","last_active":"2024-04-12T08:44:02.928803+0000","last_peered":"2024-04-12T08:44:02.928803+0000","last_clean":"2024-04-12T08:44:02.928803+0000","last_became_active":"2024-04-12T08:43:28.128830+0000","last_became_peered":"2024-04-12T08:43:28.128830+0000","last_unstale":"2024-04-12T08:44:02.928803+0000","last_undegraded":"2024-04-12T08:44:02.928803+0000","last_fullsized":"2024-04-12T08:44:02.928803+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:59:35.467533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.731170+0000","last_change":"2024-04-12T08:42:18.340361+0000","last_active":"2024-04-12T08:44:02.731170+0000","last_peered":"2024-04-12T08:44:02.731170+0000","last_clean":"2024-04-12T08:44:02.731170+0000","last_became_active":"2024-04-12T08:42:18.340050+0000","last_became_peered":"2024-04-12T08:42:18.340050+0000","last_unstale":"2024-04-12T08:44:02.731170+0000","last_undegraded":"2024-04-12T08:44:02.731170+0000","last_fullsized":"2024-04-12T08:44:02.731170+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:56:18.913365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724849+0000","last_change":"2024-04-12T08:43:28.867289+0000","last_active":"2024-04-12T08:44:02.724849+0000","last_peered":"2024-04-12T08:44:02.724849+0000","last_clean":"2024-04-12T08:44:02.724849+0000","last_became_active":"2024-04-12T08:43:28.866834+0000","last_became_peered":"2024-04-12T08:43:28.866834+0000","last_unstale":"2024-04-12T08:44:02.724849+0000","last_undegraded":"2024-04-12T08:44:02.724849+0000","last_fullsized":"2024-04-12T08:44:02.724849+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:49:59.143037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728552+0000","last_change":"2024-04-12T08:42:18.728964+0000","last_active":"2024-04-12T08:44:02.728552+0000","last_peered":"2024-04-12T08:44:02.728552+0000","last_clean":"2024-04-12T08:44:02.728552+0000","last_became_active":"2024-04-12T08:42:18.728592+0000","last_became_peered":"2024-04-12T08:42:18.728592+0000","last_unstale":"2024-04-12T08:44:02.728552+0000","last_undegraded":"2024-04-12T08:44:02.728552+0000","last_fullsized":"2024-04-12T08:44:02.728552+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:54:31.647606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725293+0000","last_change":"2024-04-12T08:43:28.130068+0000","last_active":"2024-04-12T08:44:02.725293+0000","last_peered":"2024-04-12T08:44:02.725293+0000","last_clean":"2024-04-12T08:44:02.725293+0000","last_became_active":"2024-04-12T08:43:28.129254+0000","last_became_peered":"2024-04-12T08:43:28.129254+0000","last_unstale":"2024-04-12T08:44:02.725293+0000","last_undegraded":"2024-04-12T08:44:02.725293+0000","last_fullsized":"2024-04-12T08:44:02.725293+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:04:56.032025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928597+0000","last_change":"2024-04-12T08:43:29.892281+0000","last_active":"2024-04-12T08:44:02.928597+0000","last_peered":"2024-04-12T08:44:02.928597+0000","last_clean":"2024-04-12T08:44:02.928597+0000","last_became_active":"2024-04-12T08:43:29.892025+0000","last_became_peered":"2024-04-12T08:43:29.892025+0000","last_unstale":"2024-04-12T08:44:02.928597+0000","last_undegraded":"2024-04-12T08:44:02.928597+0000","last_fullsized":"2024-04-12T08:44:02.928597+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:26:57.089643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728283+0000","last_change":"2024-04-12T08:43:28.131983+0000","last_active":"2024-04-12T08:44:02.728283+0000","last_peered":"2024-04-12T08:44:02.728283+0000","last_clean":"2024-04-12T08:44:02.728283+0000","last_became_active":"2024-04-12T08:43:28.131681+0000","last_became_peered":"2024-04-12T08:43:28.131681+0000","last_unstale":"2024-04-12T08:44:02.728283+0000","last_undegraded":"2024-04-12T08:44:02.728283+0000","last_fullsized":"2024-04-12T08:44:02.728283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:20:27.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728882+0000","last_change":"2024-04-12T08:42:18.728907+0000","last_active":"2024-04-12T08:44:02.728882+0000","last_peered":"2024-04-12T08:44:02.728882+0000","last_clean":"2024-04-12T08:44:02.728882+0000","last_became_active":"2024-04-12T08:42:18.728438+0000","last_became_peered":"2024-04-12T08:42:18.728438+0000","last_unstale":"2024-04-12T08:44:02.728882+0000","last_undegraded":"2024-04-12T08:44:02.728882+0000","last_fullsized":"2024-04-12T08:44:02.728882+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:20:28.071214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928786+0000","last_change":"2024-04-12T08:43:29.148429+0000","last_active":"2024-04-12T08:44:02.928786+0000","last_peered":"2024-04-12T08:44:02.928786+0000","last_clean":"2024-04-12T08:44:02.928786+0000","last_became_active":"2024-04-12T08:43:29.148084+0000","last_became_peered":"2024-04-12T08:43:29.148084+0000","last_unstale":"2024-04-12T08:44:02.928786+0000","last_undegraded":"2024-04-12T08:44:02.928786+0000","last_fullsized":"2024-04-12T08:44:02.928786+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:12:00.376740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928574+0000","last_change":"2024-04-12T08:43:28.131582+0000","last_active":"2024-04-12T08:44:02.928574+0000","last_peered":"2024-04-12T08:44:02.928574+0000","last_clean":"2024-04-12T08:44:02.928574+0000","last_became_active":"2024-04-12T08:43:28.131297+0000","last_became_peered":"2024-04-12T08:43:28.131297+0000","last_unstale":"2024-04-12T08:44:02.928574+0000","last_undegraded":"2024-04-12T08:44:02.928574+0000","last_fullsized":"2024-04-12T08:44:02.928574+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:22:04.612326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727793+0000","last_change":"2024-04-12T08:42:18.346705+0000","last_active":"2024-04-12T08:44:02.727793+0000","last_peered":"2024-04-12T08:44:02.727793+0000","last_clean":"2024-04-12T08:44:02.727793+0000","last_became_active":"2024-04-12T08:42:18.346382+0000","last_became_peered":"2024-04-12T08:42:18.346382+0000","last_unstale":"2024-04-12T08:44:02.727793+0000","last_undegraded":"2024-04-12T08:44:02.727793+0000","last_fullsized":"2024-04-12T08:44:02.727793+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:12:09.463598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.731156+0000","last_change":"2024-04-12T08:43:29.892386+0000","last_active":"2024-04-12T08:44:02.731156+0000","last_peered":"2024-04-12T08:44:02.731156+0000","last_clean":"2024-04-12T08:44:02.731156+0000","last_became_active":"2024-04-12T08:43:29.892091+0000","last_became_peered":"2024-04-12T08:43:29.892091+0000","last_unstale":"2024-04-12T08:44:02.731156+0000","last_undegraded":"2024-04-12T08:44:02.731156+0000","last_fullsized":"2024-04-12T08:44:02.731156+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:32:11.122269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725193+0000","last_change":"2024-04-12T08:43:28.128879+0000","last_active":"2024-04-12T08:44:02.725193+0000","last_peered":"2024-04-12T08:44:02.725193+0000","last_clean":"2024-04-12T08:44:02.725193+0000","last_became_active":"2024-04-12T08:43:28.127629+0000","last_became_peered":"2024-04-12T08:43:28.127629+0000","last_unstale":"2024-04-12T08:44:02.725193+0000","last_undegraded":"2024-04-12T08:44:02.725193+0000","last_fullsized":"2024-04-12T08:44:02.725193+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:09:20.778628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728067+0000","last_change":"2024-04-12T08:41:37.324892+0000","last_active":"2024-04-12T08:44:02.728067+0000","last_peered":"2024-04-12T08:44:02.728067+0000","last_clean":"2024-04-12T08:44:02.728067+0000","last_became_active":"2024-04-12T08:41:37.324471+0000","last_became_peered":"2024-04-12T08:41:37.324471+0000","last_unstale":"2024-04-12T08:44:02.728067+0000","last_undegraded":"2024-04-12T08:44:02.728067+0000","last_fullsized":"2024-04-12T08:44:02.728067+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_clean_scrub_stamp":"2024-04-12T08:39:39.156733+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:03:42.742838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720281+0000","last_change":"2024-04-12T08:42:18.339523+0000","last_active":"2024-04-12T08:44:02.720281+0000","last_peered":"2024-04-12T08:44:02.720281+0000","last_clean":"2024-04-12T08:44:02.720281+0000","last_became_active":"2024-04-12T08:42:18.339215+0000","last_became_peered":"2024-04-12T08:42:18.339215+0000","last_unstale":"2024-04-12T08:44:02.720281+0000","last_undegraded":"2024-04-12T08:44:02.720281+0000","last_fullsized":"2024-04-12T08:44:02.720281+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:45:58.963209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728562+0000","last_change":"2024-04-12T08:43:29.148195+0000","last_active":"2024-04-12T08:44:02.728562+0000","last_peered":"2024-04-12T08:44:02.728562+0000","last_clean":"2024-04-12T08:44:02.728562+0000","last_became_active":"2024-04-12T08:43:29.147896+0000","last_became_peered":"2024-04-12T08:43:29.147896+0000","last_unstale":"2024-04-12T08:44:02.728562+0000","last_undegraded":"2024-04-12T08:44:02.728562+0000","last_fullsized":"2024-04-12T08:44:02.728562+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:31.589995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727695+0000","last_change":"2024-04-12T08:43:28.133016+0000","last_active":"2024-04-12T08:44:02.727695+0000","last_peered":"2024-04-12T08:44:02.727695+0000","last_clean":"2024-04-12T08:44:02.727695+0000","last_became_active":"2024-04-12T08:43:28.132744+0000","last_became_peered":"2024-04-12T08:43:28.132744+0000","last_unstale":"2024-04-12T08:44:02.727695+0000","last_undegraded":"2024-04-12T08:44:02.727695+0000","last_fullsized":"2024-04-12T08:44:02.727695+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:33.135688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725111+0000","last_change":"2024-04-12T08:42:18.457898+0000","last_active":"2024-04-12T08:44:02.725111+0000","last_peered":"2024-04-12T08:44:02.725111+0000","last_clean":"2024-04-12T08:44:02.725111+0000","last_became_active":"2024-04-12T08:42:18.457597+0000","last_became_peered":"2024-04-12T08:42:18.457597+0000","last_unstale":"2024-04-12T08:44:02.725111+0000","last_undegraded":"2024-04-12T08:44:02.725111+0000","last_fullsized":"2024-04-12T08:44:02.725111+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:01:02.172496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728222+0000","last_change":"2024-04-12T08:43:29.817779+0000","last_active":"2024-04-12T08:44:02.728222+0000","last_peered":"2024-04-12T08:44:02.728222+0000","last_clean":"2024-04-12T08:44:02.728222+0000","last_became_active":"2024-04-12T08:43:29.817464+0000","last_became_peered":"2024-04-12T08:43:29.817464+0000","last_unstale":"2024-04-12T08:44:02.728222+0000","last_undegraded":"2024-04-12T08:44:02.728222+0000","last_fullsized":"2024-04-12T08:44:02.728222+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:40:56.303507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728624+0000","last_change":"2024-04-12T08:43:28.135529+0000","last_active":"2024-04-12T08:44:02.728624+0000","last_peered":"2024-04-12T08:44:02.728624+0000","last_clean":"2024-04-12T08:44:02.728624+0000","last_became_active":"2024-04-12T08:43:28.135298+0000","last_became_peered":"2024-04-12T08:43:28.135298+0000","last_unstale":"2024-04-12T08:44:02.728624+0000","last_undegraded":"2024-04-12T08:44:02.728624+0000","last_fullsized":"2024-04-12T08:44:02.728624+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:01:43.266984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720209+0000","last_change":"2024-04-12T08:42:18.340485+0000","last_active":"2024-04-12T08:44:02.720209+0000","last_peered":"2024-04-12T08:44:02.720209+0000","last_clean":"2024-04-12T08:44:02.720209+0000","last_became_active":"2024-04-12T08:42:18.340137+0000","last_became_peered":"2024-04-12T08:42:18.340137+0000","last_unstale":"2024-04-12T08:44:02.720209+0000","last_undegraded":"2024-04-12T08:44:02.720209+0000","last_fullsized":"2024-04-12T08:44:02.720209+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:05:14.634360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727690+0000","last_change":"2024-04-12T08:43:29.847797+0000","last_active":"2024-04-12T08:44:02.727690+0000","last_peered":"2024-04-12T08:44:02.727690+0000","last_clean":"2024-04-12T08:44:02.727690+0000","last_became_active":"2024-04-12T08:43:29.847493+0000","last_became_peered":"2024-04-12T08:43:29.847493+0000","last_unstale":"2024-04-12T08:44:02.727690+0000","last_undegraded":"2024-04-12T08:44:02.727690+0000","last_fullsized":"2024-04-12T08:44:02.727690+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:29:23.205732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720508+0000","last_change":"2024-04-12T08:43:28.124883+0000","last_active":"2024-04-12T08:44:02.720508+0000","last_peered":"2024-04-12T08:44:02.720508+0000","last_clean":"2024-04-12T08:44:02.720508+0000","last_became_active":"2024-04-12T08:43:28.124548+0000","last_became_peered":"2024-04-12T08:43:28.124548+0000","last_unstale":"2024-04-12T08:44:02.720508+0000","last_undegraded":"2024-04-12T08:44:02.720508+0000","last_fullsized":"2024-04-12T08:44:02.720508+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:17.597220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728193+0000","last_change":"2024-04-12T08:42:18.346660+0000","last_active":"2024-04-12T08:44:02.728193+0000","last_peered":"2024-04-12T08:44:02.728193+0000","last_clean":"2024-04-12T08:44:02.728193+0000","last_became_active":"2024-04-12T08:42:18.346313+0000","last_became_peered":"2024-04-12T08:42:18.346313+0000","last_unstale":"2024-04-12T08:44:02.728193+0000","last_undegraded":"2024-04-12T08:44:02.728193+0000","last_fullsized":"2024-04-12T08:44:02.728193+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:40.908045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-12T08:44:03.727815+0000","last_change":"2024-04-12T08:44:03.727815+0000","last_active":"2024-04-12T08:44:03.727815+0000","last_peered":"2024-04-12T08:44:03.727815+0000","last_clean":"2024-04-12T08:44:03.727815+0000","last_became_active":"2024-04-12T08:44:03.727492+0000","last_became_peered":"2024-04-12T08:44:03.727492+0000","last_unstale":"2024-04-12T08:44:03.727815+0000","last_undegraded":"2024-04-12T08:44:03.727815+0000","last_fullsized":"2024-04-12T08:44:03.727815+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:58.741428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724923+0000","last_change":"2024-04-12T08:43:28.128780+0000","last_active":"2024-04-12T08:44:02.724923+0000","last_peered":"2024-04-12T08:44:02.724923+0000","last_clean":"2024-04-12T08:44:02.724923+0000","last_became_active":"2024-04-12T08:43:28.128504+0000","last_became_peered":"2024-04-12T08:43:28.128504+0000","last_unstale":"2024-04-12T08:44:02.724923+0000","last_undegraded":"2024-04-12T08:44:02.724923+0000","last_fullsized":"2024-04-12T08:44:02.724923+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:54:38.606937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728568+0000","last_change":"2024-04-12T08:43:28.365608+0000","last_active":"2024-04-12T08:44:02.728568+0000","last_peered":"2024-04-12T08:44:02.728568+0000","last_clean":"2024-04-12T08:44:02.728568+0000","last_became_active":"2024-04-12T08:43:28.365318+0000","last_became_peered":"2024-04-12T08:43:28.365318+0000","last_unstale":"2024-04-12T08:44:02.728568+0000","last_undegraded":"2024-04-12T08:44:02.728568+0000","last_fullsized":"2024-04-12T08:44:02.728568+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:36:33.731059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728779+0000","last_change":"2024-04-12T08:43:28.128953+0000","last_active":"2024-04-12T08:44:02.728779+0000","last_peered":"2024-04-12T08:44:02.728779+0000","last_clean":"2024-04-12T08:44:02.728779+0000","last_became_active":"2024-04-12T08:43:28.128641+0000","last_became_peered":"2024-04-12T08:43:28.128641+0000","last_unstale":"2024-04-12T08:44:02.728779+0000","last_undegraded":"2024-04-12T08:44:02.728779+0000","last_fullsized":"2024-04-12T08:44:02.728779+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:08:51.213462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728581+0000","last_change":"2024-04-12T08:43:28.142468+0000","last_active":"2024-04-12T08:44:02.728581+0000","last_peered":"2024-04-12T08:44:02.728581+0000","last_clean":"2024-04-12T08:44:02.728581+0000","last_became_active":"2024-04-12T08:43:28.142219+0000","last_became_peered":"2024-04-12T08:43:28.142219+0000","last_unstale":"2024-04-12T08:44:02.728581+0000","last_undegraded":"2024-04-12T08:44:02.728581+0000","last_fullsized":"2024-04-12T08:44:02.728581+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:10.110498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.730983+0000","last_change":"2024-04-12T08:43:28.134749+0000","last_active":"2024-04-12T08:44:02.730983+0000","last_peered":"2024-04-12T08:44:02.730983+0000","last_clean":"2024-04-12T08:44:02.730983+0000","last_became_active":"2024-04-12T08:43:28.134469+0000","last_became_peered":"2024-04-12T08:43:28.134469+0000","last_unstale":"2024-04-12T08:44:02.730983+0000","last_undegraded":"2024-04-12T08:44:02.730983+0000","last_fullsized":"2024-04-12T08:44:02.730983+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:14:06.499096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728303+0000","last_change":"2024-04-12T08:43:28.364644+0000","last_active":"2024-04-12T08:44:02.728303+0000","last_peered":"2024-04-12T08:44:02.728303+0000","last_clean":"2024-04-12T08:44:02.728303+0000","last_became_active":"2024-04-12T08:43:28.364324+0000","last_became_peered":"2024-04-12T08:43:28.364324+0000","last_unstale":"2024-04-12T08:44:02.728303+0000","last_undegraded":"2024-04-12T08:44:02.728303+0000","last_fullsized":"2024-04-12T08:44:02.728303+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:34:22.541713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725324+0000","last_change":"2024-04-12T08:43:28.130091+0000","last_active":"2024-04-12T08:44:02.725324+0000","last_peered":"2024-04-12T08:44:02.725324+0000","last_clean":"2024-04-12T08:44:02.725324+0000","last_became_active":"2024-04-12T08:43:28.129100+0000","last_became_peered":"2024-04-12T08:43:28.129100+0000","last_unstale":"2024-04-12T08:44:02.725324+0000","last_undegraded":"2024-04-12T08:44:02.725324+0000","last_fullsized":"2024-04-12T08:44:02.725324+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:50:04.117621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928692+0000","last_change":"2024-04-12T08:43:29.847884+0000","last_active":"2024-04-12T08:44:02.928692+0000","last_peered":"2024-04-12T08:44:02.928692+0000","last_clean":"2024-04-12T08:44:02.928692+0000","last_became_active":"2024-04-12T08:43:29.847587+0000","last_became_peered":"2024-04-12T08:43:29.847587+0000","last_unstale":"2024-04-12T08:44:02.928692+0000","last_undegraded":"2024-04-12T08:44:02.928692+0000","last_fullsized":"2024-04-12T08:44:02.928692+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:21:29.246792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720413+0000","last_change":"2024-04-12T08:43:28.132979+0000","last_active":"2024-04-12T08:44:02.720413+0000","last_peered":"2024-04-12T08:44:02.720413+0000","last_clean":"2024-04-12T08:44:02.720413+0000","last_became_active":"2024-04-12T08:43:28.132665+0000","last_became_peered":"2024-04-12T08:43:28.132665+0000","last_unstale":"2024-04-12T08:44:02.720413+0000","last_undegraded":"2024-04-12T08:44:02.720413+0000","last_fullsized":"2024-04-12T08:44:02.720413+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:18:08.592898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.730875+0000","last_change":"2024-04-12T08:43:29.892360+0000","last_active":"2024-04-12T08:44:02.730875+0000","last_peered":"2024-04-12T08:44:02.730875+0000","last_clean":"2024-04-12T08:44:02.730875+0000","last_became_active":"2024-04-12T08:43:29.892045+0000","last_became_peered":"2024-04-12T08:43:29.892045+0000","last_unstale":"2024-04-12T08:44:02.730875+0000","last_undegraded":"2024-04-12T08:44:02.730875+0000","last_fullsized":"2024-04-12T08:44:02.730875+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:07:20.076633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728984+0000","last_change":"2024-04-12T08:43:28.133358+0000","last_active":"2024-04-12T08:44:02.728984+0000","last_peered":"2024-04-12T08:44:02.728984+0000","last_clean":"2024-04-12T08:44:02.728984+0000","last_became_active":"2024-04-12T08:43:28.133112+0000","last_became_peered":"2024-04-12T08:43:28.133112+0000","last_unstale":"2024-04-12T08:44:02.728984+0000","last_undegraded":"2024-04-12T08:44:02.728984+0000","last_fullsized":"2024-04-12T08:44:02.728984+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:29:38.251048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727986+0000","last_change":"2024-04-12T08:43:28.147420+0000","last_active":"2024-04-12T08:44:02.727986+0000","last_peered":"2024-04-12T08:44:02.727986+0000","last_clean":"2024-04-12T08:44:02.727986+0000","last_became_active":"2024-04-12T08:43:28.147152+0000","last_became_peered":"2024-04-12T08:43:28.147152+0000","last_unstale":"2024-04-12T08:44:02.727986+0000","last_undegraded":"2024-04-12T08:44:02.727986+0000","last_fullsized":"2024-04-12T08:44:02.727986+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:57:32.048963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728648+0000","last_change":"2024-04-12T08:43:28.129530+0000","last_active":"2024-04-12T08:44:02.728648+0000","last_peered":"2024-04-12T08:44:02.728648+0000","last_clean":"2024-04-12T08:44:02.728648+0000","last_became_active":"2024-04-12T08:43:28.129261+0000","last_became_peered":"2024-04-12T08:43:28.129261+0000","last_unstale":"2024-04-12T08:44:02.728648+0000","last_undegraded":"2024-04-12T08:44:02.728648+0000","last_fullsized":"2024-04-12T08:44:02.728648+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:15:56.045523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720198+0000","last_change":"2024-04-12T08:43:28.866806+0000","last_active":"2024-04-12T08:44:02.720198+0000","last_peered":"2024-04-12T08:44:02.720198+0000","last_clean":"2024-04-12T08:44:02.720198+0000","last_became_active":"2024-04-12T08:43:28.866472+0000","last_became_peered":"2024-04-12T08:43:28.866472+0000","last_unstale":"2024-04-12T08:44:02.720198+0000","last_undegraded":"2024-04-12T08:44:02.720198+0000","last_fullsized":"2024-04-12T08:44:02.720198+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:23.104376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728406+0000","last_change":"2024-04-12T08:43:28.130202+0000","last_active":"2024-04-12T08:44:02.728406+0000","last_peered":"2024-04-12T08:44:02.728406+0000","last_clean":"2024-04-12T08:44:02.728406+0000","last_became_active":"2024-04-12T08:43:28.129928+0000","last_became_peered":"2024-04-12T08:43:28.129928+0000","last_unstale":"2024-04-12T08:44:02.728406+0000","last_undegraded":"2024-04-12T08:44:02.728406+0000","last_fullsized":"2024-04-12T08:44:02.728406+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:08:54.014114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725368+0000","last_change":"2024-04-12T08:43:28.866116+0000","last_active":"2024-04-12T08:44:02.725368+0000","last_peered":"2024-04-12T08:44:02.725368+0000","last_clean":"2024-04-12T08:44:02.725368+0000","last_became_active":"2024-04-12T08:43:28.865791+0000","last_became_peered":"2024-04-12T08:43:28.865791+0000","last_unstale":"2024-04-12T08:44:02.725368+0000","last_undegraded":"2024-04-12T08:44:02.725368+0000","last_fullsized":"2024-04-12T08:44:02.725368+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:40:31.502728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.731432+0000","last_change":"2024-04-12T08:43:28.125412+0000","last_active":"2024-04-12T08:44:02.731432+0000","last_peered":"2024-04-12T08:44:02.731432+0000","last_clean":"2024-04-12T08:44:02.731432+0000","last_became_active":"2024-04-12T08:43:28.124915+0000","last_became_peered":"2024-04-12T08:43:28.124915+0000","last_unstale":"2024-04-12T08:44:02.731432+0000","last_undegraded":"2024-04-12T08:44:02.731432+0000","last_fullsized":"2024-04-12T08:44:02.731432+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:50:09.833586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928708+0000","last_change":"2024-04-12T08:43:29.893164+0000","last_active":"2024-04-12T08:44:02.928708+0000","last_peered":"2024-04-12T08:44:02.928708+0000","last_clean":"2024-04-12T08:44:02.928708+0000","last_became_active":"2024-04-12T08:43:29.892818+0000","last_became_peered":"2024-04-12T08:43:29.892818+0000","last_unstale":"2024-04-12T08:44:02.928708+0000","last_undegraded":"2024-04-12T08:44:02.928708+0000","last_fullsized":"2024-04-12T08:44:02.928708+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:32:40.041320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727645+0000","last_change":"2024-04-12T08:43:28.133594+0000","last_active":"2024-04-12T08:44:02.727645+0000","last_peered":"2024-04-12T08:44:02.727645+0000","last_clean":"2024-04-12T08:44:02.727645+0000","last_became_active":"2024-04-12T08:43:28.133305+0000","last_became_peered":"2024-04-12T08:43:28.133305+0000","last_unstale":"2024-04-12T08:44:02.727645+0000","last_undegraded":"2024-04-12T08:44:02.727645+0000","last_fullsized":"2024-04-12T08:44:02.727645+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:09:26.576479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720361+0000","last_change":"2024-04-12T08:43:28.865936+0000","last_active":"2024-04-12T08:44:02.720361+0000","last_peered":"2024-04-12T08:44:02.720361+0000","last_clean":"2024-04-12T08:44:02.720361+0000","last_became_active":"2024-04-12T08:43:28.865637+0000","last_became_peered":"2024-04-12T08:43:28.865637+0000","last_unstale":"2024-04-12T08:44:02.720361+0000","last_undegraded":"2024-04-12T08:44:02.720361+0000","last_fullsized":"2024-04-12T08:44:02.720361+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:53:47.577358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928590+0000","last_change":"2024-04-12T08:43:28.127337+0000","last_active":"2024-04-12T08:44:02.928590+0000","last_peered":"2024-04-12T08:44:02.928590+0000","last_clean":"2024-04-12T08:44:02.928590+0000","last_became_active":"2024-04-12T08:43:28.126854+0000","last_became_peered":"2024-04-12T08:43:28.126854+0000","last_unstale":"2024-04-12T08:44:02.928590+0000","last_undegraded":"2024-04-12T08:44:02.928590+0000","last_fullsized":"2024-04-12T08:44:02.928590+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:42:25.687340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-12T08:44:03.730591+0000","last_change":"2024-04-12T08:44:03.730591+0000","last_active":"2024-04-12T08:44:03.730591+0000","last_peered":"2024-04-12T08:44:03.730591+0000","last_clean":"2024-04-12T08:44:03.730591+0000","last_became_active":"2024-04-12T08:44:03.730265+0000","last_became_peered":"2024-04-12T08:44:03.730265+0000","last_unstale":"2024-04-12T08:44:03.730591+0000","last_undegraded":"2024-04-12T08:44:03.730591+0000","last_fullsized":"2024-04-12T08:44:03.730591+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:11:57.661847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728014+0000","last_change":"2024-04-12T08:43:28.131266+0000","last_active":"2024-04-12T08:44:02.728014+0000","last_peered":"2024-04-12T08:44:02.728014+0000","last_clean":"2024-04-12T08:44:02.728014+0000","last_became_active":"2024-04-12T08:43:28.131013+0000","last_became_peered":"2024-04-12T08:43:28.131013+0000","last_unstale":"2024-04-12T08:44:02.728014+0000","last_undegraded":"2024-04-12T08:44:02.728014+0000","last_fullsized":"2024-04-12T08:44:02.728014+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:48:20.785810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728911+0000","last_change":"2024-04-12T08:43:28.364662+0000","last_active":"2024-04-12T08:44:02.728911+0000","last_peered":"2024-04-12T08:44:02.728911+0000","last_clean":"2024-04-12T08:44:02.728911+0000","last_became_active":"2024-04-12T08:43:28.364393+0000","last_became_peered":"2024-04-12T08:43:28.364393+0000","last_unstale":"2024-04-12T08:44:02.728911+0000","last_undegraded":"2024-04-12T08:44:02.728911+0000","last_fullsized":"2024-04-12T08:44:02.728911+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:20:07.367392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728845+0000","last_change":"2024-04-12T08:43:28.134686+0000","last_active":"2024-04-12T08:44:02.728845+0000","last_peered":"2024-04-12T08:44:02.728845+0000","last_clean":"2024-04-12T08:44:02.728845+0000","last_became_active":"2024-04-12T08:43:28.134402+0000","last_became_peered":"2024-04-12T08:43:28.134402+0000","last_unstale":"2024-04-12T08:44:02.728845+0000","last_undegraded":"2024-04-12T08:44:02.728845+0000","last_fullsized":"2024-04-12T08:44:02.728845+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:01:50.752620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727561+0000","last_change":"2024-04-12T08:43:29.149099+0000","last_active":"2024-04-12T08:44:02.727561+0000","last_peered":"2024-04-12T08:44:02.727561+0000","last_clean":"2024-04-12T08:44:02.727561+0000","last_became_active":"2024-04-12T08:43:29.148817+0000","last_became_peered":"2024-04-12T08:43:29.148817+0000","last_unstale":"2024-04-12T08:44:02.727561+0000","last_undegraded":"2024-04-12T08:44:02.727561+0000","last_fullsized":"2024-04-12T08:44:02.727561+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:04:45.710357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728796+0000","last_change":"2024-04-12T08:43:28.130500+0000","last_active":"2024-04-12T08:44:02.728796+0000","last_peered":"2024-04-12T08:44:02.728796+0000","last_clean":"2024-04-12T08:44:02.728796+0000","last_became_active":"2024-04-12T08:43:28.130224+0000","last_became_peered":"2024-04-12T08:43:28.130224+0000","last_unstale":"2024-04-12T08:44:02.728796+0000","last_undegraded":"2024-04-12T08:44:02.728796+0000","last_fullsized":"2024-04-12T08:44:02.728796+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:04:46.734637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724858+0000","last_change":"2024-04-12T08:43:28.866158+0000","last_active":"2024-04-12T08:44:02.724858+0000","last_peered":"2024-04-12T08:44:02.724858+0000","last_clean":"2024-04-12T08:44:02.724858+0000","last_became_active":"2024-04-12T08:43:28.865833+0000","last_became_peered":"2024-04-12T08:43:28.865833+0000","last_unstale":"2024-04-12T08:44:02.724858+0000","last_undegraded":"2024-04-12T08:44:02.724858+0000","last_fullsized":"2024-04-12T08:44:02.724858+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:44:26.078641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720424+0000","last_change":"2024-04-12T08:43:28.127173+0000","last_active":"2024-04-12T08:44:02.720424+0000","last_peered":"2024-04-12T08:44:02.720424+0000","last_clean":"2024-04-12T08:44:02.720424+0000","last_became_active":"2024-04-12T08:43:28.126843+0000","last_became_peered":"2024-04-12T08:43:28.126843+0000","last_unstale":"2024-04-12T08:44:02.720424+0000","last_undegraded":"2024-04-12T08:44:02.720424+0000","last_fullsized":"2024-04-12T08:44:02.720424+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:36:01.045664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724989+0000","last_change":"2024-04-12T08:43:28.366788+0000","last_active":"2024-04-12T08:44:02.724989+0000","last_peered":"2024-04-12T08:44:02.724989+0000","last_clean":"2024-04-12T08:44:02.724989+0000","last_became_active":"2024-04-12T08:43:28.366433+0000","last_became_peered":"2024-04-12T08:43:28.366433+0000","last_unstale":"2024-04-12T08:44:02.724989+0000","last_undegraded":"2024-04-12T08:44:02.724989+0000","last_fullsized":"2024-04-12T08:44:02.724989+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:25:10.412231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928827+0000","last_change":"2024-04-12T08:43:28.134516+0000","last_active":"2024-04-12T08:44:02.928827+0000","last_peered":"2024-04-12T08:44:02.928827+0000","last_clean":"2024-04-12T08:44:02.928827+0000","last_became_active":"2024-04-12T08:43:28.134271+0000","last_became_peered":"2024-04-12T08:43:28.134271+0000","last_unstale":"2024-04-12T08:44:02.928827+0000","last_undegraded":"2024-04-12T08:44:02.928827+0000","last_fullsized":"2024-04-12T08:44:02.928827+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:26:50.562379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.731102+0000","last_change":"2024-04-12T08:43:29.149061+0000","last_active":"2024-04-12T08:44:02.731102+0000","last_peered":"2024-04-12T08:44:02.731102+0000","last_clean":"2024-04-12T08:44:02.731102+0000","last_became_active":"2024-04-12T08:43:29.148770+0000","last_became_peered":"2024-04-12T08:43:29.148770+0000","last_unstale":"2024-04-12T08:44:02.731102+0000","last_undegraded":"2024-04-12T08:44:02.731102+0000","last_fullsized":"2024-04-12T08:44:02.731102+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:58:45.327408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725067+0000","last_change":"2024-04-12T08:43:28.130912+0000","last_active":"2024-04-12T08:44:02.725067+0000","last_peered":"2024-04-12T08:44:02.725067+0000","last_clean":"2024-04-12T08:44:02.725067+0000","last_became_active":"2024-04-12T08:43:28.130599+0000","last_became_peered":"2024-04-12T08:43:28.130599+0000","last_unstale":"2024-04-12T08:44:02.725067+0000","last_undegraded":"2024-04-12T08:44:02.725067+0000","last_fullsized":"2024-04-12T08:44:02.725067+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:58:47.615199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928720+0000","last_change":"2024-04-12T08:43:29.893149+0000","last_active":"2024-04-12T08:44:02.928720+0000","last_peered":"2024-04-12T08:44:02.928720+0000","last_clean":"2024-04-12T08:44:02.928720+0000","last_became_active":"2024-04-12T08:43:29.892819+0000","last_became_peered":"2024-04-12T08:43:29.892819+0000","last_unstale":"2024-04-12T08:44:02.928720+0000","last_undegraded":"2024-04-12T08:44:02.928720+0000","last_fullsized":"2024-04-12T08:44:02.928720+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:43:24.857784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727568+0000","last_change":"2024-04-12T08:43:28.133429+0000","last_active":"2024-04-12T08:44:02.727568+0000","last_peered":"2024-04-12T08:44:02.727568+0000","last_clean":"2024-04-12T08:44:02.727568+0000","last_became_active":"2024-04-12T08:43:28.133127+0000","last_became_peered":"2024-04-12T08:43:28.133127+0000","last_unstale":"2024-04-12T08:44:02.727568+0000","last_undegraded":"2024-04-12T08:44:02.727568+0000","last_fullsized":"2024-04-12T08:44:02.727568+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:53:05.406948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724980+0000","last_change":"2024-04-12T08:43:28.147298+0000","last_active":"2024-04-12T08:44:02.724980+0000","last_peered":"2024-04-12T08:44:02.724980+0000","last_clean":"2024-04-12T08:44:02.724980+0000","last_became_active":"2024-04-12T08:43:28.147056+0000","last_became_peered":"2024-04-12T08:43:28.147056+0000","last_unstale":"2024-04-12T08:44:02.724980+0000","last_undegraded":"2024-04-12T08:44:02.724980+0000","last_fullsized":"2024-04-12T08:44:02.724980+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:43:55.631001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728513+0000","last_change":"2024-04-12T08:43:28.135186+0000","last_active":"2024-04-12T08:44:02.728513+0000","last_peered":"2024-04-12T08:44:02.728513+0000","last_clean":"2024-04-12T08:44:02.728513+0000","last_became_active":"2024-04-12T08:43:28.134862+0000","last_became_peered":"2024-04-12T08:43:28.134862+0000","last_unstale":"2024-04-12T08:44:02.728513+0000","last_undegraded":"2024-04-12T08:44:02.728513+0000","last_fullsized":"2024-04-12T08:44:02.728513+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:00:55.005186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201318,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":908282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.5,"15min":0.49399999999999999},"min":{"1min":0.33400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.69299999999999995,"5min":0.876,"15min":0.876},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.48099999999999998,"15min":0.47399999999999998},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.61399999999999999}]},{"osd":1,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.64600000000000002,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.503,"15min":0.497},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.68300000000000005}]},{"osd":2,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.38200000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80900000000000005,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.625},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.45000000000000001}]},{"osd":3,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.51900000000000002,"15min":0.51000000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.53300000000000003,"15min":0.52600000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.69399999999999995}]},{"osd":4,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.46200000000000002,"15min":0.45600000000000002},"min":{"1min":0.34699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.504},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.502,"15min":0.49099999999999999},"min":{"1min":0.316,"5min":0.308,"15min":0.308},"max":{"1min":0.79700000000000004,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.66900000000000004}]},{"osd":5,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.48499999999999999,"15min":0.47299999999999998},"min":{"1min":0.35499999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.53500000000000003,"15min":0.52600000000000002},"min":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82999999999999996,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.57699999999999996}]},{"osd":6,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.54800000000000004,"15min":0.53800000000000003},"min":{"1min":0.40999999999999998,"5min":0.372,"15min":0.372},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.57999999999999996,"15min":0.56999999999999995},"min":{"1min":0.496,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.84399999999999997,"5min":1.133,"15min":1.133},"last":0.52300000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1323008,"data_stored":907606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.33500000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.73099999999999998,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.61099999999999999,"5min":0.755,"15min":0.755},"last":0.48499999999999999}]},{"osd":1,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.52900000000000003,"15min":0.53300000000000003},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68500000000000005,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.503},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.63900000000000001,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.63400000000000001}]},{"osd":2,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.53600000000000003,"15min":0.53900000000000003},"min":{"1min":0.40999999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.58599999999999997,"5min":0.753,"15min":0.753},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.38700000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.69599999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.59499999999999997}]},{"osd":3,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.625,"15min":0.55900000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.61199999999999999,"5min":10.471,"15min":10.471},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.64700000000000002,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53600000000000003}]},{"osd":4,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.44,"15min":0.433},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.372},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.46899999999999997,"15min":0.45300000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.71199999999999997,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.55600000000000005}]},{"osd":5,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.47399999999999998,"15min":0.46899999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.68300000000000005,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.436},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.49299999999999999,"15min":0.47699999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.66500000000000004,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.72599999999999998}]},{"osd":7,"last update":"Fri Apr 12 08:43:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.50900000000000001,"15min":0.501},"min":{"1min":0.33100000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.53400000000000003,"15min":0.52400000000000002},"min":{"1min":0.36799999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.69799999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":315939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:44:24 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.35899999999999999,"15min":0.33500000000000002},"min":{"1min":0.24299999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.54800000000000004,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.378,"15min":0.34300000000000003},"min":{"1min":0.224,"5min":0.219,"15min":0.219},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.34300000000000003}]},{"osd":2,"last update":"Fri Apr 12 08:43:46 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.40000000000000002,"15min":0.41699999999999998},"min":{"1min":0.26700000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.43099999999999999,"15min":0.44800000000000001},"min":{"1min":0.27000000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57399999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.40899999999999997}]},{"osd":3,"last update":"Fri Apr 12 08:44:07 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.44500000000000001,"15min":0.47899999999999998},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.48699999999999999,"5min":0.628,"15min":0.628},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.434,"15min":0.46200000000000002},"min":{"1min":0.29199999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.58299999999999996,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.49099999999999999}]},{"osd":4,"last update":"Fri Apr 12 08:44:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54600000000000004,"15min":0.55600000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.76000000000000001,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.3580000000000001,"5min":0.71699999999999997,"15min":0.61699999999999999},"min":{"1min":0.34599999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":14.795999999999999,"5min":14.795999999999999,"15min":14.795999999999999},"last":0.60499999999999998}]},{"osd":5,"last update":"Fri Apr 12 08:43:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.68600000000000005,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58699999999999997},"min":{"1min":0.41799999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.56699999999999995}]},{"osd":6,"last update":"Fri Apr 12 08:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57199999999999995,"15min":0.56399999999999995},"min":{"1min":0.45700000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74399999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.41999999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.76400000000000001,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.61899999999999999}]},{"osd":7,"last update":"Fri Apr 12 08:43:41 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.57399999999999995,"15min":0.56899999999999995},"min":{"1min":0.44,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.57499999999999996,"15min":0.56799999999999995},"min":{"1min":0.49199999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.65400000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738440,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298648,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446504,"statfs":{"total":95995035648,"available":95689220096,"internally_reserved":0,"allocated":1323008,"data_stored":907164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 08:44:24 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35499999999999998,"15min":0.315},"min":{"1min":0.21099999999999999,"5min":0.189,"15min":0.189},"max":{"1min":0.58799999999999997,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.36199999999999999,"15min":0.32600000000000001},"min":{"1min":0.222,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.53400000000000003,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.41099999999999998}]},{"osd":2,"last update":"Fri Apr 12 08:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.379,"15min":0.379},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.52600000000000002,"5min":0.623,"15min":0.623},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.39800000000000002,"15min":0.40200000000000002},"min":{"1min":0.26600000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.55400000000000005,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.34799999999999998}]},{"osd":3,"last update":"Fri Apr 12 08:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.42299999999999999,"15min":0.44800000000000001},"min":{"1min":0.28999999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59899999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.44700000000000001,"15min":0.47499999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.65300000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.377}]},{"osd":4,"last update":"Fri Apr 12 08:43:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.58199999999999996,"15min":0.58699999999999997},"min":{"1min":0.41799999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.80400000000000005,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.58999999999999997,"15min":0.60399999999999998},"min":{"1min":0.35399999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.76200000000000001,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.58899999999999997}]},{"osd":5,"last update":"Fri Apr 12 08:43:52 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.58599999999999997,"15min":0.60599999999999998},"min":{"1min":0.43099999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69899999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.61299999999999999,"15min":0.623},"min":{"1min":0.434,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78000000000000003,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.54500000000000004}]},{"osd":6,"last update":"Fri Apr 12 08:44:19 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.60199999999999998,"15min":0.61699999999999999},"min":{"1min":0.38400000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.72199999999999998,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.61199999999999999,"15min":0.626},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.74099999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.63500000000000001}]},{"osd":7,"last update":"Fri Apr 12 08:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.60299999999999998,"15min":0.59999999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.82099999999999995,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.622,"5min":0.59799999999999998,"15min":0.59399999999999997},"min":{"1min":0.498,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.79300000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.61299999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":733184,"data_stored":317073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:43:44 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36699999999999999,"15min":0.35899999999999999},"min":{"1min":0.23100000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.374,"5min":0.39300000000000002,"15min":0.38800000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.58799999999999997,"5min":0.629,"15min":0.629},"last":0.40400000000000003}]},{"osd":1,"last update":"Fri Apr 12 08:43:44 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.379,"15min":0.371},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.59299999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.372},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.41399999999999998,"15min":0.39500000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.65400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.438}]},{"osd":3,"last update":"Fri Apr 12 08:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.46000000000000002,"15min":0.47999999999999998},"min":{"1min":0.34300000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.58999999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.443,"15min":0.46700000000000003},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57399999999999995,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.42899999999999999}]},{"osd":4,"last update":"Fri Apr 12 08:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.90700000000000003,"15min":1.23},"min":{"1min":0.42499999999999999,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.88,"5min":14.991,"15min":14.991},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.60899999999999999,"15min":0.60299999999999998},"min":{"1min":0.441,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.621}]},{"osd":5,"last update":"Fri Apr 12 08:43:53 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.64500000000000002,"15min":0.66100000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82199999999999995,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63600000000000001,"15min":0.64500000000000002},"min":{"1min":0.47099999999999997,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84899999999999998,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.70099999999999996}]},{"osd":6,"last update":"Fri Apr 12 08:44:16 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.63,"15min":0.63},"min":{"1min":0.438,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.94699999999999995,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.90000000000000002,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.68999999999999995}]},{"osd":7,"last update":"Fri Apr 12 08:43:35 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.61599999999999999,"15min":0.61199999999999999},"min":{"1min":0.46000000000000002,"5min":0.438,"15min":0.438},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.60899999999999999,"15min":0.60099999999999998},"min":{"1min":0.48799999999999999,"5min":0.439,"15min":0.439},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.71199999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":316559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.371,"15min":0.38600000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.50900000000000001,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.39900000000000002,"15min":0.42699999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.46300000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.37}]},{"osd":1,"last update":"Fri Apr 12 08:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.41599999999999998,"15min":0.44900000000000001},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.56599999999999995,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.317},{"interface":"front","average":{"1min":0.371,"5min":0.42399999999999999,"15min":0.46200000000000002},"min":{"1min":0.26000000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.59899999999999998,"5min":0.751,"15min":0.751},"last":0.25700000000000001}]},{"osd":2,"last update":"Fri Apr 12 08:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.44500000000000001,"15min":0.48199999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.49099999999999999,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.38,"5min":0.41999999999999998,"15min":0.44700000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.55300000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.29799999999999999}]},{"osd":4,"last update":"Fri Apr 12 08:44:27 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58099999999999996,"15min":0.59399999999999997},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.58599999999999997,"15min":0.59799999999999998},"min":{"1min":0.36799999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.81799999999999995,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.621}]},{"osd":5,"last update":"Fri Apr 12 08:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.59099999999999997,"15min":0.59999999999999998},"min":{"1min":0.40000000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.59399999999999997,"15min":0.60799999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.80500000000000005,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.57199999999999995}]},{"osd":6,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.58599999999999997,"15min":0.58799999999999997},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57799999999999996,"15min":0.57099999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.50800000000000001}]},{"osd":7,"last update":"Fri Apr 12 08:43:37 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.59399999999999997,"15min":0.58699999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.59799999999999998,"15min":0.58599999999999997},"min":{"1min":0.52900000000000003,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.80900000000000005,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.60099999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986228,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298088,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447064,"statfs":{"total":95995035648,"available":95689793536,"internally_reserved":0,"allocated":749568,"data_stored":318167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.68600000000000005,"5min":0.877,"15min":0.877},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53300000000000003,"15min":0.53200000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67200000000000004,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.72499999999999998}]},{"osd":1,"last update":"Fri Apr 12 08:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.53000000000000003,"15min":0.53600000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.70599999999999996,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.73699999999999999,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.53500000000000003}]},{"osd":2,"last update":"Fri Apr 12 08:43:29 2024","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":0.72199999999999998,"15min":0.60099999999999998},"min":{"1min":0.38600000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":17.882999999999999,"5min":17.882999999999999,"15min":17.882999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.71999999999999997,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.67400000000000004}]},{"osd":3,"last update":"Fri Apr 12 08:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.379,"5min":0.377,"15min":0.377},"max":{"1min":0.75600000000000001,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.54800000000000004,"15min":0.54700000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.69299999999999995,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.64700000000000002}]},{"osd":5,"last update":"Fri Apr 12 08:43:52 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.40699999999999997,"15min":0.38500000000000001},"min":{"1min":0.34999999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.44400000000000001,"15min":0.42599999999999999},"min":{"1min":0.316,"5min":0.309,"15min":0.309},"max":{"1min":0.81200000000000006,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.56999999999999995}]},{"osd":6,"last update":"Fri Apr 12 08:44:14 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.433,"15min":0.40500000000000003},"min":{"1min":0.311,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.72699999999999998,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.51900000000000002,"15min":0.49299999999999999},"min":{"1min":0.40999999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.78100000000000003,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.65800000000000003}]},{"osd":7,"last update":"Fri Apr 12 08:43:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.40699999999999997,"5min":0.37,"15min":0.37},"max":{"1min":0.77100000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.41799999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.751,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.59799999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724591,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":733184,"data_stored":317109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:43:56 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.51800000000000002,"15min":0.52800000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52800000000000002,"15min":0.52600000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.74299999999999999,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.57899999999999996}]},{"osd":1,"last update":"Fri Apr 12 08:43:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.53900000000000003,"15min":0.53800000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.74199999999999999,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.627},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.51700000000000002,"15min":0.52500000000000002},"min":{"1min":0.36499999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.64700000000000002,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.56000000000000005}]},{"osd":2,"last update":"Fri Apr 12 08:43:56 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.52600000000000002,"15min":0.53100000000000003},"min":{"1min":0.35999999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.437,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.73099999999999998,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.53000000000000003}]},{"osd":3,"last update":"Fri Apr 12 08:43:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.66800000000000004,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.57399999999999995,"15min":0.56799999999999995},"min":{"1min":0.435,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.71599999999999997}]},{"osd":4,"last update":"Fri Apr 12 08:43:56 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.41299999999999998,"15min":0.39700000000000002},"min":{"1min":0.317,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.435,"15min":0.40899999999999997},"min":{"1min":0.36899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.67300000000000004,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.72899999999999998}]},{"osd":6,"last update":"Fri Apr 12 08:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.45700000000000002,"15min":0.435},"min":{"1min":0.38900000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.61799999999999999,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.496,"15min":0.46300000000000002},"min":{"1min":0.374,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.755,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.66800000000000004}]},{"osd":7,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.51000000000000001,"15min":0.48799999999999999},"min":{"1min":0.46500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.753},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.5,"15min":0.48599999999999999},"min":{"1min":0.44700000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.72499999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.65800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T08:44:30.805 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T08:44:31.059 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:32.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:31 smithi114 ceph-mon[33573]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:44:32.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:31 smithi114 ceph-mon[33573]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:32.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:31 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:31] "GET /metrics HTTP/1.1" 200 239256 "" "Prometheus/2.43.0" 2024-04-12T08:44:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:31 smithi090 ceph-mon[40003]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:44:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:31 smithi090 ceph-mon[40003]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:32.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:31 smithi090 ceph-mon[29100]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:44:32.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:31 smithi090 ceph-mon[29100]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:32.907 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:32 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:32.908 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:32 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:33.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:32 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:33.207 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:44:33.209 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-12T08:44:33.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:33 smithi090 ceph-mon[40003]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:33.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:33 smithi090 ceph-mon[29100]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:34.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:33 smithi114 ceph-mon[33573]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:34.153 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":89,"stamp":"2024-04-12T08:44:32.626823+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386336,"kb_used_data":7472,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574880,"statfs":{"total":767960285184,"available":765516677120,"internally_reserved":0,"allocated":7651328,"data_stored":4307899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002727"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-12T08:44:03.895417+0000","last_change":"2024-04-12T08:44:03.895417+0000","last_active":"2024-04-12T08:44:03.895417+0000","last_peered":"2024-04-12T08:44:03.895417+0000","last_clean":"2024-04-12T08:44:03.895417+0000","last_became_active":"2024-04-12T08:44:03.895040+0000","last_became_peered":"2024-04-12T08:44:03.895040+0000","last_unstale":"2024-04-12T08:44:03.895417+0000","last_undegraded":"2024-04-12T08:44:03.895417+0000","last_fullsized":"2024-04-12T08:44:03.895417+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:23:30.820364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928856+0000","last_change":"2024-04-12T08:43:28.126923+0000","last_active":"2024-04-12T08:44:02.928856+0000","last_peered":"2024-04-12T08:44:02.928856+0000","last_clean":"2024-04-12T08:44:02.928856+0000","last_became_active":"2024-04-12T08:43:28.126605+0000","last_became_peered":"2024-04-12T08:43:28.126605+0000","last_unstale":"2024-04-12T08:44:02.928856+0000","last_undegraded":"2024-04-12T08:44:02.928856+0000","last_fullsized":"2024-04-12T08:44:02.928856+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:53:04.343113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724730+0000","last_change":"2024-04-12T08:43:28.319152+0000","last_active":"2024-04-12T08:44:02.724730+0000","last_peered":"2024-04-12T08:44:02.724730+0000","last_clean":"2024-04-12T08:44:02.724730+0000","last_became_active":"2024-04-12T08:43:28.318772+0000","last_became_peered":"2024-04-12T08:43:28.318772+0000","last_unstale":"2024-04-12T08:44:02.724730+0000","last_undegraded":"2024-04-12T08:44:02.724730+0000","last_fullsized":"2024-04-12T08:44:02.724730+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:11:08.333819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725586+0000","last_change":"2024-04-12T08:43:28.128857+0000","last_active":"2024-04-12T08:44:02.725586+0000","last_peered":"2024-04-12T08:44:02.725586+0000","last_clean":"2024-04-12T08:44:02.725586+0000","last_became_active":"2024-04-12T08:43:28.127170+0000","last_became_peered":"2024-04-12T08:43:28.127170+0000","last_unstale":"2024-04-12T08:44:02.725586+0000","last_undegraded":"2024-04-12T08:44:02.725586+0000","last_fullsized":"2024-04-12T08:44:02.725586+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:11:07.335293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728465+0000","last_change":"2024-04-12T08:43:28.319145+0000","last_active":"2024-04-12T08:44:02.728465+0000","last_peered":"2024-04-12T08:44:02.728465+0000","last_clean":"2024-04-12T08:44:02.728465+0000","last_became_active":"2024-04-12T08:43:28.318896+0000","last_became_peered":"2024-04-12T08:43:28.318896+0000","last_unstale":"2024-04-12T08:44:02.728465+0000","last_undegraded":"2024-04-12T08:44:02.728465+0000","last_fullsized":"2024-04-12T08:44:02.728465+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:04:09.427910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720474+0000","last_change":"2024-04-12T08:43:28.137332+0000","last_active":"2024-04-12T08:44:02.720474+0000","last_peered":"2024-04-12T08:44:02.720474+0000","last_clean":"2024-04-12T08:44:02.720474+0000","last_became_active":"2024-04-12T08:43:28.136749+0000","last_became_peered":"2024-04-12T08:43:28.136749+0000","last_unstale":"2024-04-12T08:44:02.720474+0000","last_undegraded":"2024-04-12T08:44:02.720474+0000","last_fullsized":"2024-04-12T08:44:02.720474+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:20:09.223627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728131+0000","last_change":"2024-04-12T08:43:28.147487+0000","last_active":"2024-04-12T08:44:02.728131+0000","last_peered":"2024-04-12T08:44:02.728131+0000","last_clean":"2024-04-12T08:44:02.728131+0000","last_became_active":"2024-04-12T08:43:28.147265+0000","last_became_peered":"2024-04-12T08:43:28.147265+0000","last_unstale":"2024-04-12T08:44:02.728131+0000","last_undegraded":"2024-04-12T08:44:02.728131+0000","last_fullsized":"2024-04-12T08:44:02.728131+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:40:54.441361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720466+0000","last_change":"2024-04-12T08:43:28.131575+0000","last_active":"2024-04-12T08:44:02.720466+0000","last_peered":"2024-04-12T08:44:02.720466+0000","last_clean":"2024-04-12T08:44:02.720466+0000","last_became_active":"2024-04-12T08:43:28.131277+0000","last_became_peered":"2024-04-12T08:43:28.131277+0000","last_unstale":"2024-04-12T08:44:02.720466+0000","last_undegraded":"2024-04-12T08:44:02.720466+0000","last_fullsized":"2024-04-12T08:44:02.720466+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:22:44.474309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724560+0000","last_change":"2024-04-12T08:43:28.147262+0000","last_active":"2024-04-12T08:44:02.724560+0000","last_peered":"2024-04-12T08:44:02.724560+0000","last_clean":"2024-04-12T08:44:02.724560+0000","last_became_active":"2024-04-12T08:43:28.146959+0000","last_became_peered":"2024-04-12T08:43:28.146959+0000","last_unstale":"2024-04-12T08:44:02.724560+0000","last_undegraded":"2024-04-12T08:44:02.724560+0000","last_fullsized":"2024-04-12T08:44:02.724560+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:04:51.802195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928770+0000","last_change":"2024-04-12T08:43:28.129883+0000","last_active":"2024-04-12T08:44:02.928770+0000","last_peered":"2024-04-12T08:44:02.928770+0000","last_clean":"2024-04-12T08:44:02.928770+0000","last_became_active":"2024-04-12T08:43:28.129633+0000","last_became_peered":"2024-04-12T08:43:28.129633+0000","last_unstale":"2024-04-12T08:44:02.928770+0000","last_undegraded":"2024-04-12T08:44:02.928770+0000","last_fullsized":"2024-04-12T08:44:02.928770+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:13:47.677812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728867+0000","last_change":"2024-04-12T08:43:28.365741+0000","last_active":"2024-04-12T08:44:02.728867+0000","last_peered":"2024-04-12T08:44:02.728867+0000","last_clean":"2024-04-12T08:44:02.728867+0000","last_became_active":"2024-04-12T08:43:28.365425+0000","last_became_peered":"2024-04-12T08:43:28.365425+0000","last_unstale":"2024-04-12T08:44:02.728867+0000","last_undegraded":"2024-04-12T08:44:02.728867+0000","last_fullsized":"2024-04-12T08:44:02.728867+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:13:45.303498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928803+0000","last_change":"2024-04-12T08:43:28.129089+0000","last_active":"2024-04-12T08:44:02.928803+0000","last_peered":"2024-04-12T08:44:02.928803+0000","last_clean":"2024-04-12T08:44:02.928803+0000","last_became_active":"2024-04-12T08:43:28.128830+0000","last_became_peered":"2024-04-12T08:43:28.128830+0000","last_unstale":"2024-04-12T08:44:02.928803+0000","last_undegraded":"2024-04-12T08:44:02.928803+0000","last_fullsized":"2024-04-12T08:44:02.928803+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:59:35.467533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.731170+0000","last_change":"2024-04-12T08:42:18.340361+0000","last_active":"2024-04-12T08:44:02.731170+0000","last_peered":"2024-04-12T08:44:02.731170+0000","last_clean":"2024-04-12T08:44:02.731170+0000","last_became_active":"2024-04-12T08:42:18.340050+0000","last_became_peered":"2024-04-12T08:42:18.340050+0000","last_unstale":"2024-04-12T08:44:02.731170+0000","last_undegraded":"2024-04-12T08:44:02.731170+0000","last_fullsized":"2024-04-12T08:44:02.731170+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:56:18.913365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724849+0000","last_change":"2024-04-12T08:43:28.867289+0000","last_active":"2024-04-12T08:44:02.724849+0000","last_peered":"2024-04-12T08:44:02.724849+0000","last_clean":"2024-04-12T08:44:02.724849+0000","last_became_active":"2024-04-12T08:43:28.866834+0000","last_became_peered":"2024-04-12T08:43:28.866834+0000","last_unstale":"2024-04-12T08:44:02.724849+0000","last_undegraded":"2024-04-12T08:44:02.724849+0000","last_fullsized":"2024-04-12T08:44:02.724849+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:49:59.143037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728552+0000","last_change":"2024-04-12T08:42:18.728964+0000","last_active":"2024-04-12T08:44:02.728552+0000","last_peered":"2024-04-12T08:44:02.728552+0000","last_clean":"2024-04-12T08:44:02.728552+0000","last_became_active":"2024-04-12T08:42:18.728592+0000","last_became_peered":"2024-04-12T08:42:18.728592+0000","last_unstale":"2024-04-12T08:44:02.728552+0000","last_undegraded":"2024-04-12T08:44:02.728552+0000","last_fullsized":"2024-04-12T08:44:02.728552+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:54:31.647606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725293+0000","last_change":"2024-04-12T08:43:28.130068+0000","last_active":"2024-04-12T08:44:02.725293+0000","last_peered":"2024-04-12T08:44:02.725293+0000","last_clean":"2024-04-12T08:44:02.725293+0000","last_became_active":"2024-04-12T08:43:28.129254+0000","last_became_peered":"2024-04-12T08:43:28.129254+0000","last_unstale":"2024-04-12T08:44:02.725293+0000","last_undegraded":"2024-04-12T08:44:02.725293+0000","last_fullsized":"2024-04-12T08:44:02.725293+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:04:56.032025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928597+0000","last_change":"2024-04-12T08:43:29.892281+0000","last_active":"2024-04-12T08:44:02.928597+0000","last_peered":"2024-04-12T08:44:02.928597+0000","last_clean":"2024-04-12T08:44:02.928597+0000","last_became_active":"2024-04-12T08:43:29.892025+0000","last_became_peered":"2024-04-12T08:43:29.892025+0000","last_unstale":"2024-04-12T08:44:02.928597+0000","last_undegraded":"2024-04-12T08:44:02.928597+0000","last_fullsized":"2024-04-12T08:44:02.928597+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:26:57.089643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728283+0000","last_change":"2024-04-12T08:43:28.131983+0000","last_active":"2024-04-12T08:44:02.728283+0000","last_peered":"2024-04-12T08:44:02.728283+0000","last_clean":"2024-04-12T08:44:02.728283+0000","last_became_active":"2024-04-12T08:43:28.131681+0000","last_became_peered":"2024-04-12T08:43:28.131681+0000","last_unstale":"2024-04-12T08:44:02.728283+0000","last_undegraded":"2024-04-12T08:44:02.728283+0000","last_fullsized":"2024-04-12T08:44:02.728283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:20:27.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728882+0000","last_change":"2024-04-12T08:42:18.728907+0000","last_active":"2024-04-12T08:44:02.728882+0000","last_peered":"2024-04-12T08:44:02.728882+0000","last_clean":"2024-04-12T08:44:02.728882+0000","last_became_active":"2024-04-12T08:42:18.728438+0000","last_became_peered":"2024-04-12T08:42:18.728438+0000","last_unstale":"2024-04-12T08:44:02.728882+0000","last_undegraded":"2024-04-12T08:44:02.728882+0000","last_fullsized":"2024-04-12T08:44:02.728882+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:20:28.071214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928786+0000","last_change":"2024-04-12T08:43:29.148429+0000","last_active":"2024-04-12T08:44:02.928786+0000","last_peered":"2024-04-12T08:44:02.928786+0000","last_clean":"2024-04-12T08:44:02.928786+0000","last_became_active":"2024-04-12T08:43:29.148084+0000","last_became_peered":"2024-04-12T08:43:29.148084+0000","last_unstale":"2024-04-12T08:44:02.928786+0000","last_undegraded":"2024-04-12T08:44:02.928786+0000","last_fullsized":"2024-04-12T08:44:02.928786+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:12:00.376740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928574+0000","last_change":"2024-04-12T08:43:28.131582+0000","last_active":"2024-04-12T08:44:02.928574+0000","last_peered":"2024-04-12T08:44:02.928574+0000","last_clean":"2024-04-12T08:44:02.928574+0000","last_became_active":"2024-04-12T08:43:28.131297+0000","last_became_peered":"2024-04-12T08:43:28.131297+0000","last_unstale":"2024-04-12T08:44:02.928574+0000","last_undegraded":"2024-04-12T08:44:02.928574+0000","last_fullsized":"2024-04-12T08:44:02.928574+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:22:04.612326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727793+0000","last_change":"2024-04-12T08:42:18.346705+0000","last_active":"2024-04-12T08:44:02.727793+0000","last_peered":"2024-04-12T08:44:02.727793+0000","last_clean":"2024-04-12T08:44:02.727793+0000","last_became_active":"2024-04-12T08:42:18.346382+0000","last_became_peered":"2024-04-12T08:42:18.346382+0000","last_unstale":"2024-04-12T08:44:02.727793+0000","last_undegraded":"2024-04-12T08:44:02.727793+0000","last_fullsized":"2024-04-12T08:44:02.727793+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:12:09.463598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.731156+0000","last_change":"2024-04-12T08:43:29.892386+0000","last_active":"2024-04-12T08:44:02.731156+0000","last_peered":"2024-04-12T08:44:02.731156+0000","last_clean":"2024-04-12T08:44:02.731156+0000","last_became_active":"2024-04-12T08:43:29.892091+0000","last_became_peered":"2024-04-12T08:43:29.892091+0000","last_unstale":"2024-04-12T08:44:02.731156+0000","last_undegraded":"2024-04-12T08:44:02.731156+0000","last_fullsized":"2024-04-12T08:44:02.731156+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:32:11.122269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725193+0000","last_change":"2024-04-12T08:43:28.128879+0000","last_active":"2024-04-12T08:44:02.725193+0000","last_peered":"2024-04-12T08:44:02.725193+0000","last_clean":"2024-04-12T08:44:02.725193+0000","last_became_active":"2024-04-12T08:43:28.127629+0000","last_became_peered":"2024-04-12T08:43:28.127629+0000","last_unstale":"2024-04-12T08:44:02.725193+0000","last_undegraded":"2024-04-12T08:44:02.725193+0000","last_fullsized":"2024-04-12T08:44:02.725193+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:09:20.778628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728067+0000","last_change":"2024-04-12T08:41:37.324892+0000","last_active":"2024-04-12T08:44:02.728067+0000","last_peered":"2024-04-12T08:44:02.728067+0000","last_clean":"2024-04-12T08:44:02.728067+0000","last_became_active":"2024-04-12T08:41:37.324471+0000","last_became_peered":"2024-04-12T08:41:37.324471+0000","last_unstale":"2024-04-12T08:44:02.728067+0000","last_undegraded":"2024-04-12T08:44:02.728067+0000","last_fullsized":"2024-04-12T08:44:02.728067+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_clean_scrub_stamp":"2024-04-12T08:39:39.156733+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:03:42.742838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720281+0000","last_change":"2024-04-12T08:42:18.339523+0000","last_active":"2024-04-12T08:44:02.720281+0000","last_peered":"2024-04-12T08:44:02.720281+0000","last_clean":"2024-04-12T08:44:02.720281+0000","last_became_active":"2024-04-12T08:42:18.339215+0000","last_became_peered":"2024-04-12T08:42:18.339215+0000","last_unstale":"2024-04-12T08:44:02.720281+0000","last_undegraded":"2024-04-12T08:44:02.720281+0000","last_fullsized":"2024-04-12T08:44:02.720281+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:45:58.963209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728562+0000","last_change":"2024-04-12T08:43:29.148195+0000","last_active":"2024-04-12T08:44:02.728562+0000","last_peered":"2024-04-12T08:44:02.728562+0000","last_clean":"2024-04-12T08:44:02.728562+0000","last_became_active":"2024-04-12T08:43:29.147896+0000","last_became_peered":"2024-04-12T08:43:29.147896+0000","last_unstale":"2024-04-12T08:44:02.728562+0000","last_undegraded":"2024-04-12T08:44:02.728562+0000","last_fullsized":"2024-04-12T08:44:02.728562+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:31.589995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727695+0000","last_change":"2024-04-12T08:43:28.133016+0000","last_active":"2024-04-12T08:44:02.727695+0000","last_peered":"2024-04-12T08:44:02.727695+0000","last_clean":"2024-04-12T08:44:02.727695+0000","last_became_active":"2024-04-12T08:43:28.132744+0000","last_became_peered":"2024-04-12T08:43:28.132744+0000","last_unstale":"2024-04-12T08:44:02.727695+0000","last_undegraded":"2024-04-12T08:44:02.727695+0000","last_fullsized":"2024-04-12T08:44:02.727695+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:33.135688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725111+0000","last_change":"2024-04-12T08:42:18.457898+0000","last_active":"2024-04-12T08:44:02.725111+0000","last_peered":"2024-04-12T08:44:02.725111+0000","last_clean":"2024-04-12T08:44:02.725111+0000","last_became_active":"2024-04-12T08:42:18.457597+0000","last_became_peered":"2024-04-12T08:42:18.457597+0000","last_unstale":"2024-04-12T08:44:02.725111+0000","last_undegraded":"2024-04-12T08:44:02.725111+0000","last_fullsized":"2024-04-12T08:44:02.725111+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:01:02.172496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728222+0000","last_change":"2024-04-12T08:43:29.817779+0000","last_active":"2024-04-12T08:44:02.728222+0000","last_peered":"2024-04-12T08:44:02.728222+0000","last_clean":"2024-04-12T08:44:02.728222+0000","last_became_active":"2024-04-12T08:43:29.817464+0000","last_became_peered":"2024-04-12T08:43:29.817464+0000","last_unstale":"2024-04-12T08:44:02.728222+0000","last_undegraded":"2024-04-12T08:44:02.728222+0000","last_fullsized":"2024-04-12T08:44:02.728222+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:40:56.303507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728624+0000","last_change":"2024-04-12T08:43:28.135529+0000","last_active":"2024-04-12T08:44:02.728624+0000","last_peered":"2024-04-12T08:44:02.728624+0000","last_clean":"2024-04-12T08:44:02.728624+0000","last_became_active":"2024-04-12T08:43:28.135298+0000","last_became_peered":"2024-04-12T08:43:28.135298+0000","last_unstale":"2024-04-12T08:44:02.728624+0000","last_undegraded":"2024-04-12T08:44:02.728624+0000","last_fullsized":"2024-04-12T08:44:02.728624+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:01:43.266984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720209+0000","last_change":"2024-04-12T08:42:18.340485+0000","last_active":"2024-04-12T08:44:02.720209+0000","last_peered":"2024-04-12T08:44:02.720209+0000","last_clean":"2024-04-12T08:44:02.720209+0000","last_became_active":"2024-04-12T08:42:18.340137+0000","last_became_peered":"2024-04-12T08:42:18.340137+0000","last_unstale":"2024-04-12T08:44:02.720209+0000","last_undegraded":"2024-04-12T08:44:02.720209+0000","last_fullsized":"2024-04-12T08:44:02.720209+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:05:14.634360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727690+0000","last_change":"2024-04-12T08:43:29.847797+0000","last_active":"2024-04-12T08:44:02.727690+0000","last_peered":"2024-04-12T08:44:02.727690+0000","last_clean":"2024-04-12T08:44:02.727690+0000","last_became_active":"2024-04-12T08:43:29.847493+0000","last_became_peered":"2024-04-12T08:43:29.847493+0000","last_unstale":"2024-04-12T08:44:02.727690+0000","last_undegraded":"2024-04-12T08:44:02.727690+0000","last_fullsized":"2024-04-12T08:44:02.727690+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:29:23.205732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720508+0000","last_change":"2024-04-12T08:43:28.124883+0000","last_active":"2024-04-12T08:44:02.720508+0000","last_peered":"2024-04-12T08:44:02.720508+0000","last_clean":"2024-04-12T08:44:02.720508+0000","last_became_active":"2024-04-12T08:43:28.124548+0000","last_became_peered":"2024-04-12T08:43:28.124548+0000","last_unstale":"2024-04-12T08:44:02.720508+0000","last_undegraded":"2024-04-12T08:44:02.720508+0000","last_fullsized":"2024-04-12T08:44:02.720508+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:17.597220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728193+0000","last_change":"2024-04-12T08:42:18.346660+0000","last_active":"2024-04-12T08:44:02.728193+0000","last_peered":"2024-04-12T08:44:02.728193+0000","last_clean":"2024-04-12T08:44:02.728193+0000","last_became_active":"2024-04-12T08:42:18.346313+0000","last_became_peered":"2024-04-12T08:42:18.346313+0000","last_unstale":"2024-04-12T08:44:02.728193+0000","last_undegraded":"2024-04-12T08:44:02.728193+0000","last_fullsized":"2024-04-12T08:44:02.728193+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:40.908045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-12T08:44:03.727815+0000","last_change":"2024-04-12T08:44:03.727815+0000","last_active":"2024-04-12T08:44:03.727815+0000","last_peered":"2024-04-12T08:44:03.727815+0000","last_clean":"2024-04-12T08:44:03.727815+0000","last_became_active":"2024-04-12T08:44:03.727492+0000","last_became_peered":"2024-04-12T08:44:03.727492+0000","last_unstale":"2024-04-12T08:44:03.727815+0000","last_undegraded":"2024-04-12T08:44:03.727815+0000","last_fullsized":"2024-04-12T08:44:03.727815+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:58.741428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724923+0000","last_change":"2024-04-12T08:43:28.128780+0000","last_active":"2024-04-12T08:44:02.724923+0000","last_peered":"2024-04-12T08:44:02.724923+0000","last_clean":"2024-04-12T08:44:02.724923+0000","last_became_active":"2024-04-12T08:43:28.128504+0000","last_became_peered":"2024-04-12T08:43:28.128504+0000","last_unstale":"2024-04-12T08:44:02.724923+0000","last_undegraded":"2024-04-12T08:44:02.724923+0000","last_fullsized":"2024-04-12T08:44:02.724923+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:54:38.606937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728568+0000","last_change":"2024-04-12T08:43:28.365608+0000","last_active":"2024-04-12T08:44:02.728568+0000","last_peered":"2024-04-12T08:44:02.728568+0000","last_clean":"2024-04-12T08:44:02.728568+0000","last_became_active":"2024-04-12T08:43:28.365318+0000","last_became_peered":"2024-04-12T08:43:28.365318+0000","last_unstale":"2024-04-12T08:44:02.728568+0000","last_undegraded":"2024-04-12T08:44:02.728568+0000","last_fullsized":"2024-04-12T08:44:02.728568+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:36:33.731059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728779+0000","last_change":"2024-04-12T08:43:28.128953+0000","last_active":"2024-04-12T08:44:02.728779+0000","last_peered":"2024-04-12T08:44:02.728779+0000","last_clean":"2024-04-12T08:44:02.728779+0000","last_became_active":"2024-04-12T08:43:28.128641+0000","last_became_peered":"2024-04-12T08:43:28.128641+0000","last_unstale":"2024-04-12T08:44:02.728779+0000","last_undegraded":"2024-04-12T08:44:02.728779+0000","last_fullsized":"2024-04-12T08:44:02.728779+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:08:51.213462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728581+0000","last_change":"2024-04-12T08:43:28.142468+0000","last_active":"2024-04-12T08:44:02.728581+0000","last_peered":"2024-04-12T08:44:02.728581+0000","last_clean":"2024-04-12T08:44:02.728581+0000","last_became_active":"2024-04-12T08:43:28.142219+0000","last_became_peered":"2024-04-12T08:43:28.142219+0000","last_unstale":"2024-04-12T08:44:02.728581+0000","last_undegraded":"2024-04-12T08:44:02.728581+0000","last_fullsized":"2024-04-12T08:44:02.728581+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:10.110498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.730983+0000","last_change":"2024-04-12T08:43:28.134749+0000","last_active":"2024-04-12T08:44:02.730983+0000","last_peered":"2024-04-12T08:44:02.730983+0000","last_clean":"2024-04-12T08:44:02.730983+0000","last_became_active":"2024-04-12T08:43:28.134469+0000","last_became_peered":"2024-04-12T08:43:28.134469+0000","last_unstale":"2024-04-12T08:44:02.730983+0000","last_undegraded":"2024-04-12T08:44:02.730983+0000","last_fullsized":"2024-04-12T08:44:02.730983+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:14:06.499096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728303+0000","last_change":"2024-04-12T08:43:28.364644+0000","last_active":"2024-04-12T08:44:02.728303+0000","last_peered":"2024-04-12T08:44:02.728303+0000","last_clean":"2024-04-12T08:44:02.728303+0000","last_became_active":"2024-04-12T08:43:28.364324+0000","last_became_peered":"2024-04-12T08:43:28.364324+0000","last_unstale":"2024-04-12T08:44:02.728303+0000","last_undegraded":"2024-04-12T08:44:02.728303+0000","last_fullsized":"2024-04-12T08:44:02.728303+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:34:22.541713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725324+0000","last_change":"2024-04-12T08:43:28.130091+0000","last_active":"2024-04-12T08:44:02.725324+0000","last_peered":"2024-04-12T08:44:02.725324+0000","last_clean":"2024-04-12T08:44:02.725324+0000","last_became_active":"2024-04-12T08:43:28.129100+0000","last_became_peered":"2024-04-12T08:43:28.129100+0000","last_unstale":"2024-04-12T08:44:02.725324+0000","last_undegraded":"2024-04-12T08:44:02.725324+0000","last_fullsized":"2024-04-12T08:44:02.725324+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:50:04.117621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928692+0000","last_change":"2024-04-12T08:43:29.847884+0000","last_active":"2024-04-12T08:44:02.928692+0000","last_peered":"2024-04-12T08:44:02.928692+0000","last_clean":"2024-04-12T08:44:02.928692+0000","last_became_active":"2024-04-12T08:43:29.847587+0000","last_became_peered":"2024-04-12T08:43:29.847587+0000","last_unstale":"2024-04-12T08:44:02.928692+0000","last_undegraded":"2024-04-12T08:44:02.928692+0000","last_fullsized":"2024-04-12T08:44:02.928692+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:21:29.246792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720413+0000","last_change":"2024-04-12T08:43:28.132979+0000","last_active":"2024-04-12T08:44:02.720413+0000","last_peered":"2024-04-12T08:44:02.720413+0000","last_clean":"2024-04-12T08:44:02.720413+0000","last_became_active":"2024-04-12T08:43:28.132665+0000","last_became_peered":"2024-04-12T08:43:28.132665+0000","last_unstale":"2024-04-12T08:44:02.720413+0000","last_undegraded":"2024-04-12T08:44:02.720413+0000","last_fullsized":"2024-04-12T08:44:02.720413+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:18:08.592898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.730875+0000","last_change":"2024-04-12T08:43:29.892360+0000","last_active":"2024-04-12T08:44:02.730875+0000","last_peered":"2024-04-12T08:44:02.730875+0000","last_clean":"2024-04-12T08:44:02.730875+0000","last_became_active":"2024-04-12T08:43:29.892045+0000","last_became_peered":"2024-04-12T08:43:29.892045+0000","last_unstale":"2024-04-12T08:44:02.730875+0000","last_undegraded":"2024-04-12T08:44:02.730875+0000","last_fullsized":"2024-04-12T08:44:02.730875+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:07:20.076633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728984+0000","last_change":"2024-04-12T08:43:28.133358+0000","last_active":"2024-04-12T08:44:02.728984+0000","last_peered":"2024-04-12T08:44:02.728984+0000","last_clean":"2024-04-12T08:44:02.728984+0000","last_became_active":"2024-04-12T08:43:28.133112+0000","last_became_peered":"2024-04-12T08:43:28.133112+0000","last_unstale":"2024-04-12T08:44:02.728984+0000","last_undegraded":"2024-04-12T08:44:02.728984+0000","last_fullsized":"2024-04-12T08:44:02.728984+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:29:38.251048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727986+0000","last_change":"2024-04-12T08:43:28.147420+0000","last_active":"2024-04-12T08:44:02.727986+0000","last_peered":"2024-04-12T08:44:02.727986+0000","last_clean":"2024-04-12T08:44:02.727986+0000","last_became_active":"2024-04-12T08:43:28.147152+0000","last_became_peered":"2024-04-12T08:43:28.147152+0000","last_unstale":"2024-04-12T08:44:02.727986+0000","last_undegraded":"2024-04-12T08:44:02.727986+0000","last_fullsized":"2024-04-12T08:44:02.727986+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:57:32.048963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728648+0000","last_change":"2024-04-12T08:43:28.129530+0000","last_active":"2024-04-12T08:44:02.728648+0000","last_peered":"2024-04-12T08:44:02.728648+0000","last_clean":"2024-04-12T08:44:02.728648+0000","last_became_active":"2024-04-12T08:43:28.129261+0000","last_became_peered":"2024-04-12T08:43:28.129261+0000","last_unstale":"2024-04-12T08:44:02.728648+0000","last_undegraded":"2024-04-12T08:44:02.728648+0000","last_fullsized":"2024-04-12T08:44:02.728648+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:15:56.045523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720198+0000","last_change":"2024-04-12T08:43:28.866806+0000","last_active":"2024-04-12T08:44:02.720198+0000","last_peered":"2024-04-12T08:44:02.720198+0000","last_clean":"2024-04-12T08:44:02.720198+0000","last_became_active":"2024-04-12T08:43:28.866472+0000","last_became_peered":"2024-04-12T08:43:28.866472+0000","last_unstale":"2024-04-12T08:44:02.720198+0000","last_undegraded":"2024-04-12T08:44:02.720198+0000","last_fullsized":"2024-04-12T08:44:02.720198+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:23.104376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728406+0000","last_change":"2024-04-12T08:43:28.130202+0000","last_active":"2024-04-12T08:44:02.728406+0000","last_peered":"2024-04-12T08:44:02.728406+0000","last_clean":"2024-04-12T08:44:02.728406+0000","last_became_active":"2024-04-12T08:43:28.129928+0000","last_became_peered":"2024-04-12T08:43:28.129928+0000","last_unstale":"2024-04-12T08:44:02.728406+0000","last_undegraded":"2024-04-12T08:44:02.728406+0000","last_fullsized":"2024-04-12T08:44:02.728406+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:08:54.014114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725368+0000","last_change":"2024-04-12T08:43:28.866116+0000","last_active":"2024-04-12T08:44:02.725368+0000","last_peered":"2024-04-12T08:44:02.725368+0000","last_clean":"2024-04-12T08:44:02.725368+0000","last_became_active":"2024-04-12T08:43:28.865791+0000","last_became_peered":"2024-04-12T08:43:28.865791+0000","last_unstale":"2024-04-12T08:44:02.725368+0000","last_undegraded":"2024-04-12T08:44:02.725368+0000","last_fullsized":"2024-04-12T08:44:02.725368+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:40:31.502728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.731432+0000","last_change":"2024-04-12T08:43:28.125412+0000","last_active":"2024-04-12T08:44:02.731432+0000","last_peered":"2024-04-12T08:44:02.731432+0000","last_clean":"2024-04-12T08:44:02.731432+0000","last_became_active":"2024-04-12T08:43:28.124915+0000","last_became_peered":"2024-04-12T08:43:28.124915+0000","last_unstale":"2024-04-12T08:44:02.731432+0000","last_undegraded":"2024-04-12T08:44:02.731432+0000","last_fullsized":"2024-04-12T08:44:02.731432+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:50:09.833586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928708+0000","last_change":"2024-04-12T08:43:29.893164+0000","last_active":"2024-04-12T08:44:02.928708+0000","last_peered":"2024-04-12T08:44:02.928708+0000","last_clean":"2024-04-12T08:44:02.928708+0000","last_became_active":"2024-04-12T08:43:29.892818+0000","last_became_peered":"2024-04-12T08:43:29.892818+0000","last_unstale":"2024-04-12T08:44:02.928708+0000","last_undegraded":"2024-04-12T08:44:02.928708+0000","last_fullsized":"2024-04-12T08:44:02.928708+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:32:40.041320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727645+0000","last_change":"2024-04-12T08:43:28.133594+0000","last_active":"2024-04-12T08:44:02.727645+0000","last_peered":"2024-04-12T08:44:02.727645+0000","last_clean":"2024-04-12T08:44:02.727645+0000","last_became_active":"2024-04-12T08:43:28.133305+0000","last_became_peered":"2024-04-12T08:43:28.133305+0000","last_unstale":"2024-04-12T08:44:02.727645+0000","last_undegraded":"2024-04-12T08:44:02.727645+0000","last_fullsized":"2024-04-12T08:44:02.727645+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:09:26.576479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720361+0000","last_change":"2024-04-12T08:43:28.865936+0000","last_active":"2024-04-12T08:44:02.720361+0000","last_peered":"2024-04-12T08:44:02.720361+0000","last_clean":"2024-04-12T08:44:02.720361+0000","last_became_active":"2024-04-12T08:43:28.865637+0000","last_became_peered":"2024-04-12T08:43:28.865637+0000","last_unstale":"2024-04-12T08:44:02.720361+0000","last_undegraded":"2024-04-12T08:44:02.720361+0000","last_fullsized":"2024-04-12T08:44:02.720361+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:53:47.577358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928590+0000","last_change":"2024-04-12T08:43:28.127337+0000","last_active":"2024-04-12T08:44:02.928590+0000","last_peered":"2024-04-12T08:44:02.928590+0000","last_clean":"2024-04-12T08:44:02.928590+0000","last_became_active":"2024-04-12T08:43:28.126854+0000","last_became_peered":"2024-04-12T08:43:28.126854+0000","last_unstale":"2024-04-12T08:44:02.928590+0000","last_undegraded":"2024-04-12T08:44:02.928590+0000","last_fullsized":"2024-04-12T08:44:02.928590+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:42:25.687340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-12T08:44:03.730591+0000","last_change":"2024-04-12T08:44:03.730591+0000","last_active":"2024-04-12T08:44:03.730591+0000","last_peered":"2024-04-12T08:44:03.730591+0000","last_clean":"2024-04-12T08:44:03.730591+0000","last_became_active":"2024-04-12T08:44:03.730265+0000","last_became_peered":"2024-04-12T08:44:03.730265+0000","last_unstale":"2024-04-12T08:44:03.730591+0000","last_undegraded":"2024-04-12T08:44:03.730591+0000","last_fullsized":"2024-04-12T08:44:03.730591+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:11:57.661847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728014+0000","last_change":"2024-04-12T08:43:28.131266+0000","last_active":"2024-04-12T08:44:02.728014+0000","last_peered":"2024-04-12T08:44:02.728014+0000","last_clean":"2024-04-12T08:44:02.728014+0000","last_became_active":"2024-04-12T08:43:28.131013+0000","last_became_peered":"2024-04-12T08:43:28.131013+0000","last_unstale":"2024-04-12T08:44:02.728014+0000","last_undegraded":"2024-04-12T08:44:02.728014+0000","last_fullsized":"2024-04-12T08:44:02.728014+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:48:20.785810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728911+0000","last_change":"2024-04-12T08:43:28.364662+0000","last_active":"2024-04-12T08:44:02.728911+0000","last_peered":"2024-04-12T08:44:02.728911+0000","last_clean":"2024-04-12T08:44:02.728911+0000","last_became_active":"2024-04-12T08:43:28.364393+0000","last_became_peered":"2024-04-12T08:43:28.364393+0000","last_unstale":"2024-04-12T08:44:02.728911+0000","last_undegraded":"2024-04-12T08:44:02.728911+0000","last_fullsized":"2024-04-12T08:44:02.728911+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:20:07.367392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728845+0000","last_change":"2024-04-12T08:43:28.134686+0000","last_active":"2024-04-12T08:44:02.728845+0000","last_peered":"2024-04-12T08:44:02.728845+0000","last_clean":"2024-04-12T08:44:02.728845+0000","last_became_active":"2024-04-12T08:43:28.134402+0000","last_became_peered":"2024-04-12T08:43:28.134402+0000","last_unstale":"2024-04-12T08:44:02.728845+0000","last_undegraded":"2024-04-12T08:44:02.728845+0000","last_fullsized":"2024-04-12T08:44:02.728845+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:01:50.752620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727561+0000","last_change":"2024-04-12T08:43:29.149099+0000","last_active":"2024-04-12T08:44:02.727561+0000","last_peered":"2024-04-12T08:44:02.727561+0000","last_clean":"2024-04-12T08:44:02.727561+0000","last_became_active":"2024-04-12T08:43:29.148817+0000","last_became_peered":"2024-04-12T08:43:29.148817+0000","last_unstale":"2024-04-12T08:44:02.727561+0000","last_undegraded":"2024-04-12T08:44:02.727561+0000","last_fullsized":"2024-04-12T08:44:02.727561+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:04:45.710357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728796+0000","last_change":"2024-04-12T08:43:28.130500+0000","last_active":"2024-04-12T08:44:02.728796+0000","last_peered":"2024-04-12T08:44:02.728796+0000","last_clean":"2024-04-12T08:44:02.728796+0000","last_became_active":"2024-04-12T08:43:28.130224+0000","last_became_peered":"2024-04-12T08:43:28.130224+0000","last_unstale":"2024-04-12T08:44:02.728796+0000","last_undegraded":"2024-04-12T08:44:02.728796+0000","last_fullsized":"2024-04-12T08:44:02.728796+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:04:46.734637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724858+0000","last_change":"2024-04-12T08:43:28.866158+0000","last_active":"2024-04-12T08:44:02.724858+0000","last_peered":"2024-04-12T08:44:02.724858+0000","last_clean":"2024-04-12T08:44:02.724858+0000","last_became_active":"2024-04-12T08:43:28.865833+0000","last_became_peered":"2024-04-12T08:43:28.865833+0000","last_unstale":"2024-04-12T08:44:02.724858+0000","last_undegraded":"2024-04-12T08:44:02.724858+0000","last_fullsized":"2024-04-12T08:44:02.724858+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:44:26.078641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.720424+0000","last_change":"2024-04-12T08:43:28.127173+0000","last_active":"2024-04-12T08:44:02.720424+0000","last_peered":"2024-04-12T08:44:02.720424+0000","last_clean":"2024-04-12T08:44:02.720424+0000","last_became_active":"2024-04-12T08:43:28.126843+0000","last_became_peered":"2024-04-12T08:43:28.126843+0000","last_unstale":"2024-04-12T08:44:02.720424+0000","last_undegraded":"2024-04-12T08:44:02.720424+0000","last_fullsized":"2024-04-12T08:44:02.720424+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:36:01.045664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724989+0000","last_change":"2024-04-12T08:43:28.366788+0000","last_active":"2024-04-12T08:44:02.724989+0000","last_peered":"2024-04-12T08:44:02.724989+0000","last_clean":"2024-04-12T08:44:02.724989+0000","last_became_active":"2024-04-12T08:43:28.366433+0000","last_became_peered":"2024-04-12T08:43:28.366433+0000","last_unstale":"2024-04-12T08:44:02.724989+0000","last_undegraded":"2024-04-12T08:44:02.724989+0000","last_fullsized":"2024-04-12T08:44:02.724989+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:25:10.412231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928827+0000","last_change":"2024-04-12T08:43:28.134516+0000","last_active":"2024-04-12T08:44:02.928827+0000","last_peered":"2024-04-12T08:44:02.928827+0000","last_clean":"2024-04-12T08:44:02.928827+0000","last_became_active":"2024-04-12T08:43:28.134271+0000","last_became_peered":"2024-04-12T08:43:28.134271+0000","last_unstale":"2024-04-12T08:44:02.928827+0000","last_undegraded":"2024-04-12T08:44:02.928827+0000","last_fullsized":"2024-04-12T08:44:02.928827+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:26:50.562379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.731102+0000","last_change":"2024-04-12T08:43:29.149061+0000","last_active":"2024-04-12T08:44:02.731102+0000","last_peered":"2024-04-12T08:44:02.731102+0000","last_clean":"2024-04-12T08:44:02.731102+0000","last_became_active":"2024-04-12T08:43:29.148770+0000","last_became_peered":"2024-04-12T08:43:29.148770+0000","last_unstale":"2024-04-12T08:44:02.731102+0000","last_undegraded":"2024-04-12T08:44:02.731102+0000","last_fullsized":"2024-04-12T08:44:02.731102+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:58:45.327408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.725067+0000","last_change":"2024-04-12T08:43:28.130912+0000","last_active":"2024-04-12T08:44:02.725067+0000","last_peered":"2024-04-12T08:44:02.725067+0000","last_clean":"2024-04-12T08:44:02.725067+0000","last_became_active":"2024-04-12T08:43:28.130599+0000","last_became_peered":"2024-04-12T08:43:28.130599+0000","last_unstale":"2024-04-12T08:44:02.725067+0000","last_undegraded":"2024-04-12T08:44:02.725067+0000","last_fullsized":"2024-04-12T08:44:02.725067+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:58:47.615199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.928720+0000","last_change":"2024-04-12T08:43:29.893149+0000","last_active":"2024-04-12T08:44:02.928720+0000","last_peered":"2024-04-12T08:44:02.928720+0000","last_clean":"2024-04-12T08:44:02.928720+0000","last_became_active":"2024-04-12T08:43:29.892819+0000","last_became_peered":"2024-04-12T08:43:29.892819+0000","last_unstale":"2024-04-12T08:44:02.928720+0000","last_undegraded":"2024-04-12T08:44:02.928720+0000","last_fullsized":"2024-04-12T08:44:02.928720+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:43:24.857784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.727568+0000","last_change":"2024-04-12T08:43:28.133429+0000","last_active":"2024-04-12T08:44:02.727568+0000","last_peered":"2024-04-12T08:44:02.727568+0000","last_clean":"2024-04-12T08:44:02.727568+0000","last_became_active":"2024-04-12T08:43:28.133127+0000","last_became_peered":"2024-04-12T08:43:28.133127+0000","last_unstale":"2024-04-12T08:44:02.727568+0000","last_undegraded":"2024-04-12T08:44:02.727568+0000","last_fullsized":"2024-04-12T08:44:02.727568+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:53:05.406948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.724980+0000","last_change":"2024-04-12T08:43:28.147298+0000","last_active":"2024-04-12T08:44:02.724980+0000","last_peered":"2024-04-12T08:44:02.724980+0000","last_clean":"2024-04-12T08:44:02.724980+0000","last_became_active":"2024-04-12T08:43:28.147056+0000","last_became_peered":"2024-04-12T08:43:28.147056+0000","last_unstale":"2024-04-12T08:44:02.724980+0000","last_undegraded":"2024-04-12T08:44:02.724980+0000","last_fullsized":"2024-04-12T08:44:02.724980+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:43:55.631001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T08:44:02.728513+0000","last_change":"2024-04-12T08:43:28.135186+0000","last_active":"2024-04-12T08:44:02.728513+0000","last_peered":"2024-04-12T08:44:02.728513+0000","last_clean":"2024-04-12T08:44:02.728513+0000","last_became_active":"2024-04-12T08:43:28.134862+0000","last_became_peered":"2024-04-12T08:43:28.134862+0000","last_unstale":"2024-04-12T08:44:02.728513+0000","last_undegraded":"2024-04-12T08:44:02.728513+0000","last_fullsized":"2024-04-12T08:44:02.728513+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:00:55.005186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201318,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":908282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.5,"15min":0.49399999999999999},"min":{"1min":0.33400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.69299999999999995,"5min":0.876,"15min":0.876},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.48099999999999998,"15min":0.47399999999999998},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.61399999999999999}]},{"osd":1,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.64600000000000002,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.503,"15min":0.497},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.68300000000000005}]},{"osd":2,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.38200000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80900000000000005,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.625},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.45000000000000001}]},{"osd":3,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.51900000000000002,"15min":0.51000000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.53300000000000003,"15min":0.52600000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.69399999999999995}]},{"osd":4,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.46200000000000002,"15min":0.45600000000000002},"min":{"1min":0.34699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.504},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.502,"15min":0.49099999999999999},"min":{"1min":0.316,"5min":0.308,"15min":0.308},"max":{"1min":0.79700000000000004,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.66900000000000004}]},{"osd":5,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.48499999999999999,"15min":0.47299999999999998},"min":{"1min":0.35499999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.53500000000000003,"15min":0.52600000000000002},"min":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82999999999999996,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.57699999999999996}]},{"osd":6,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.54800000000000004,"15min":0.53800000000000003},"min":{"1min":0.40999999999999998,"5min":0.372,"15min":0.372},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.57999999999999996,"15min":0.56999999999999995},"min":{"1min":0.496,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.84399999999999997,"5min":1.133,"15min":1.133},"last":0.52300000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1323008,"data_stored":907606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.33500000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.73099999999999998,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.61099999999999999,"5min":0.755,"15min":0.755},"last":0.48499999999999999}]},{"osd":1,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.52900000000000003,"15min":0.53300000000000003},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68500000000000005,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.503},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.63900000000000001,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.63400000000000001}]},{"osd":2,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.53600000000000003,"15min":0.53900000000000003},"min":{"1min":0.40999999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.58599999999999997,"5min":0.753,"15min":0.753},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.38700000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.69599999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.59499999999999997}]},{"osd":3,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.625,"15min":0.55900000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.61199999999999999,"5min":10.471,"15min":10.471},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.64700000000000002,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53600000000000003}]},{"osd":4,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.44,"15min":0.433},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.372},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.46899999999999997,"15min":0.45300000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.71199999999999997,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.55600000000000005}]},{"osd":5,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.47399999999999998,"15min":0.46899999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.68300000000000005,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.436},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.49299999999999999,"15min":0.47699999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.66500000000000004,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.72599999999999998}]},{"osd":7,"last update":"Fri Apr 12 08:43:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.50900000000000001,"15min":0.501},"min":{"1min":0.33100000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.53400000000000003,"15min":0.52400000000000002},"min":{"1min":0.36799999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.69799999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":315939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:44:24 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.35899999999999999,"15min":0.33500000000000002},"min":{"1min":0.24299999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.54800000000000004,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.378,"15min":0.34300000000000003},"min":{"1min":0.224,"5min":0.219,"15min":0.219},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.34300000000000003}]},{"osd":2,"last update":"Fri Apr 12 08:43:46 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.40000000000000002,"15min":0.41699999999999998},"min":{"1min":0.26700000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.43099999999999999,"15min":0.44800000000000001},"min":{"1min":0.27000000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57399999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.40899999999999997}]},{"osd":3,"last update":"Fri Apr 12 08:44:07 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.44500000000000001,"15min":0.47899999999999998},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.48699999999999999,"5min":0.628,"15min":0.628},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.434,"15min":0.46200000000000002},"min":{"1min":0.29199999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.58299999999999996,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.49099999999999999}]},{"osd":4,"last update":"Fri Apr 12 08:44:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54600000000000004,"15min":0.55600000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.76000000000000001,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.3580000000000001,"5min":0.71699999999999997,"15min":0.61699999999999999},"min":{"1min":0.34599999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":14.795999999999999,"5min":14.795999999999999,"15min":14.795999999999999},"last":0.60499999999999998}]},{"osd":5,"last update":"Fri Apr 12 08:43:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.68600000000000005,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58699999999999997},"min":{"1min":0.41799999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.56699999999999995}]},{"osd":6,"last update":"Fri Apr 12 08:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57199999999999995,"15min":0.56399999999999995},"min":{"1min":0.45700000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74399999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.41999999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.76400000000000001,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.61899999999999999}]},{"osd":7,"last update":"Fri Apr 12 08:43:41 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.57399999999999995,"15min":0.56899999999999995},"min":{"1min":0.44,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.57499999999999996,"15min":0.56799999999999995},"min":{"1min":0.49199999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.65400000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738440,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298648,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446504,"statfs":{"total":95995035648,"available":95689220096,"internally_reserved":0,"allocated":1323008,"data_stored":907164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 08:44:24 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35499999999999998,"15min":0.315},"min":{"1min":0.21099999999999999,"5min":0.189,"15min":0.189},"max":{"1min":0.58799999999999997,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.36199999999999999,"15min":0.32600000000000001},"min":{"1min":0.222,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.53400000000000003,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.41099999999999998}]},{"osd":2,"last update":"Fri Apr 12 08:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.379,"15min":0.379},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.52600000000000002,"5min":0.623,"15min":0.623},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.39800000000000002,"15min":0.40200000000000002},"min":{"1min":0.26600000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.55400000000000005,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.34799999999999998}]},{"osd":3,"last update":"Fri Apr 12 08:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.42299999999999999,"15min":0.44800000000000001},"min":{"1min":0.28999999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59899999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.44700000000000001,"15min":0.47499999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.65300000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.377}]},{"osd":4,"last update":"Fri Apr 12 08:43:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.58199999999999996,"15min":0.58699999999999997},"min":{"1min":0.41799999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.80400000000000005,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.58999999999999997,"15min":0.60399999999999998},"min":{"1min":0.35399999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.76200000000000001,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.58899999999999997}]},{"osd":5,"last update":"Fri Apr 12 08:43:52 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.58599999999999997,"15min":0.60599999999999998},"min":{"1min":0.43099999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69899999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.61299999999999999,"15min":0.623},"min":{"1min":0.434,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78000000000000003,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.54500000000000004}]},{"osd":6,"last update":"Fri Apr 12 08:44:19 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.60199999999999998,"15min":0.61699999999999999},"min":{"1min":0.38400000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.72199999999999998,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.61199999999999999,"15min":0.626},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.74099999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.63500000000000001}]},{"osd":7,"last update":"Fri Apr 12 08:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.60299999999999998,"15min":0.59999999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.82099999999999995,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.622,"5min":0.59799999999999998,"15min":0.59399999999999997},"min":{"1min":0.498,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.79300000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.61299999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":733184,"data_stored":317073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:43:44 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36699999999999999,"15min":0.35899999999999999},"min":{"1min":0.23100000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.374,"5min":0.39300000000000002,"15min":0.38800000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.58799999999999997,"5min":0.629,"15min":0.629},"last":0.40400000000000003}]},{"osd":1,"last update":"Fri Apr 12 08:43:44 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.379,"15min":0.371},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.59299999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.372},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.41399999999999998,"15min":0.39500000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.65400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.438}]},{"osd":3,"last update":"Fri Apr 12 08:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.46000000000000002,"15min":0.47999999999999998},"min":{"1min":0.34300000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.58999999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.443,"15min":0.46700000000000003},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57399999999999995,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.42899999999999999}]},{"osd":4,"last update":"Fri Apr 12 08:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.90700000000000003,"15min":1.23},"min":{"1min":0.42499999999999999,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.88,"5min":14.991,"15min":14.991},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.60899999999999999,"15min":0.60299999999999998},"min":{"1min":0.441,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.621}]},{"osd":5,"last update":"Fri Apr 12 08:43:53 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.64500000000000002,"15min":0.66100000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82199999999999995,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63600000000000001,"15min":0.64500000000000002},"min":{"1min":0.47099999999999997,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84899999999999998,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.70099999999999996}]},{"osd":6,"last update":"Fri Apr 12 08:44:16 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.63,"15min":0.63},"min":{"1min":0.438,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.94699999999999995,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.90000000000000002,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.68999999999999995}]},{"osd":7,"last update":"Fri Apr 12 08:43:35 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.61599999999999999,"15min":0.61199999999999999},"min":{"1min":0.46000000000000002,"5min":0.438,"15min":0.438},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.60899999999999999,"15min":0.60099999999999998},"min":{"1min":0.48799999999999999,"5min":0.439,"15min":0.439},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.71199999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":316559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.371,"15min":0.38600000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.50900000000000001,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.39900000000000002,"15min":0.42699999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.46300000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.37}]},{"osd":1,"last update":"Fri Apr 12 08:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.41599999999999998,"15min":0.44900000000000001},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.56599999999999995,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.317},{"interface":"front","average":{"1min":0.371,"5min":0.42399999999999999,"15min":0.46200000000000002},"min":{"1min":0.26000000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.59899999999999998,"5min":0.751,"15min":0.751},"last":0.25700000000000001}]},{"osd":2,"last update":"Fri Apr 12 08:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.44500000000000001,"15min":0.48199999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.49099999999999999,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.38,"5min":0.41999999999999998,"15min":0.44700000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.55300000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.29799999999999999}]},{"osd":4,"last update":"Fri Apr 12 08:44:27 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58099999999999996,"15min":0.59399999999999997},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.58599999999999997,"15min":0.59799999999999998},"min":{"1min":0.36799999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.81799999999999995,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.621}]},{"osd":5,"last update":"Fri Apr 12 08:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.59099999999999997,"15min":0.59999999999999998},"min":{"1min":0.40000000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.59399999999999997,"15min":0.60799999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.80500000000000005,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.57199999999999995}]},{"osd":6,"last update":"Fri Apr 12 08:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.58599999999999997,"15min":0.58799999999999997},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57799999999999996,"15min":0.57099999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.50800000000000001}]},{"osd":7,"last update":"Fri Apr 12 08:43:37 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.59399999999999997,"15min":0.58699999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.59799999999999998,"15min":0.58599999999999997},"min":{"1min":0.52900000000000003,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.80900000000000005,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.60099999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986228,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298088,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447064,"statfs":{"total":95995035648,"available":95689793536,"internally_reserved":0,"allocated":749568,"data_stored":318167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.68600000000000005,"5min":0.877,"15min":0.877},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53300000000000003,"15min":0.53200000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67200000000000004,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.72499999999999998}]},{"osd":1,"last update":"Fri Apr 12 08:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.53000000000000003,"15min":0.53600000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.70599999999999996,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.73699999999999999,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.53500000000000003}]},{"osd":2,"last update":"Fri Apr 12 08:43:29 2024","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":0.72199999999999998,"15min":0.60099999999999998},"min":{"1min":0.38600000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":17.882999999999999,"5min":17.882999999999999,"15min":17.882999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.71999999999999997,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.67400000000000004}]},{"osd":3,"last update":"Fri Apr 12 08:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.379,"5min":0.377,"15min":0.377},"max":{"1min":0.75600000000000001,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.54800000000000004,"15min":0.54700000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.69299999999999995,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.64700000000000002}]},{"osd":5,"last update":"Fri Apr 12 08:43:52 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.40699999999999997,"15min":0.38500000000000001},"min":{"1min":0.34999999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.44400000000000001,"15min":0.42599999999999999},"min":{"1min":0.316,"5min":0.309,"15min":0.309},"max":{"1min":0.81200000000000006,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.56999999999999995}]},{"osd":6,"last update":"Fri Apr 12 08:44:14 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.433,"15min":0.40500000000000003},"min":{"1min":0.311,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.72699999999999998,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.51900000000000002,"15min":0.49299999999999999},"min":{"1min":0.40999999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.78100000000000003,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.65800000000000003}]},{"osd":7,"last update":"Fri Apr 12 08:43:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.40699999999999997,"5min":0.37,"15min":0.37},"max":{"1min":0.77100000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.41799999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.751,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.59799999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724591,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":733184,"data_stored":317109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 08:43:56 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.51800000000000002,"15min":0.52800000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52800000000000002,"15min":0.52600000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.74299999999999999,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.57899999999999996}]},{"osd":1,"last update":"Fri Apr 12 08:43:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.53900000000000003,"15min":0.53800000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.74199999999999999,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.627},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.51700000000000002,"15min":0.52500000000000002},"min":{"1min":0.36499999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.64700000000000002,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.56000000000000005}]},{"osd":2,"last update":"Fri Apr 12 08:43:56 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.52600000000000002,"15min":0.53100000000000003},"min":{"1min":0.35999999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.437,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.73099999999999998,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.53000000000000003}]},{"osd":3,"last update":"Fri Apr 12 08:43:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.66800000000000004,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.57399999999999995,"15min":0.56799999999999995},"min":{"1min":0.435,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.71599999999999997}]},{"osd":4,"last update":"Fri Apr 12 08:43:56 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.41299999999999998,"15min":0.39700000000000002},"min":{"1min":0.317,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.435,"15min":0.40899999999999997},"min":{"1min":0.36899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.67300000000000004,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.72899999999999998}]},{"osd":6,"last update":"Fri Apr 12 08:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.45700000000000002,"15min":0.435},"min":{"1min":0.38900000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.61799999999999999,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.496,"15min":0.46300000000000002},"min":{"1min":0.374,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.755,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.66800000000000004}]},{"osd":7,"last update":"Fri Apr 12 08:43:38 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.51000000000000001,"15min":0.48799999999999999},"min":{"1min":0.46500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.753},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.5,"15min":0.48599999999999999},"min":{"1min":0.44700000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.72499999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.65800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T08:44:34.157 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-12T08:44:34.157 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-12T08:44:34.157 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-12T08:44:34.158 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T08:44:34.416 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:35.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:34 smithi114 ceph-mon[33573]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:44:35.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:44:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:44:35.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:34 smithi090 ceph-mon[40003]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:44:35.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:34 smithi090 ceph-mon[29100]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T08:44:35.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:35 smithi090 ceph-mon[40003]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:35 smithi090 ceph-mon[29100]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:35 smithi114 ceph-mon[33573]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:36.840 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T08:44:36.840 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-12T08:44:37.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:37 smithi090 ceph-mon[29100]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:37.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:37 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2026838842' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T08:44:37.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:37 smithi090 ceph-mon[40003]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:37.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:37 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2026838842' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T08:44:37.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:37 smithi114 ceph-mon[33573]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:37.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:37 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2026838842' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T08:44:37.955 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-12T08:44:37.955 INFO:teuthology.run_tasks:Running task print... 2024-04-12T08:44:37.969 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-12T08:44:37.969 INFO:teuthology.run_tasks:Running task parallel... 2024-04-12T08:44:38.005 INFO:teuthology.task.parallel:starting parallel... 2024-04-12T08:44:38.005 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-12T08:44:38.005 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-04-12T08:44:38.006 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-12T08:44:38.007 INFO:tasks.radosbench:Beginning radosbench... 2024-04-12T08:44:38.008 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-04-12T08:44:38.008 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-04-12T08:44:38.008 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-12T08:44:38.008 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-12T08:44:38.009 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi090.front.sepia.ceph.com 2024-04-12T08:44:38.009 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-12T08:44:38.293 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:38.294 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:39.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:39 smithi090 ceph-mon[40003]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:39.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:39 smithi090 ceph-mon[29100]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:40.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:39 smithi114 ceph-mon[33573]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:41.879 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'unique_pool_0' created 2024-04-12T08:44:42.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:41 smithi114 ceph-mon[33573]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:42.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:41 smithi090 ceph-mon[40003]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:42.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:41 smithi090 ceph-mon[29100]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:42.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:41] "GET /metrics HTTP/1.1" 200 239244 "" "Prometheus/2.43.0" 2024-04-12T08:44:43.063 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:42 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/314192038' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-12T08:44:43.063 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:42 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/314192038' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-12T08:44:43.063 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:42 smithi090 ceph-mon[40003]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T08:44:43.064 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:42 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/314192038' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-12T08:44:43.064 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:42 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/314192038' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-12T08:44:43.064 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:42 smithi090 ceph-mon[29100]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T08:44:43.064 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:42 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:44:42] ENGINE Bus STOPPING 2024-04-12T08:44:43.064 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:42 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:44:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T08:44:43.064 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:42 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:44:42] ENGINE Bus STOPPED 2024-04-12T08:44:43.064 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:42 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:44:42] ENGINE Bus STARTING 2024-04-12T08:44:43.064 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:44:43] ENGINE Serving on http://:::9283 2024-04-12T08:44:43.064 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:44:43] ENGINE Bus STARTED 2024-04-12T08:44:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:42 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/314192038' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-12T08:44:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:42 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/314192038' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-12T08:44:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:42 smithi114 ceph-mon[33573]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T08:44:43.303 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-04-12T08:44:43.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[40003]: pgmap v95: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:43.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[40003]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:44:43.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[40003]: osdmap e73: 8 total, 8 up, 8 in 2024-04-12T08:44:43.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' 2024-04-12T08:44:43.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:44:43.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:44:43.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:44:43.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[29100]: pgmap v95: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[29100]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:44:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[29100]: osdmap e73: 8 total, 8 up, 8 in 2024-04-12T08:44:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' 2024-04-12T08:44:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:44:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:44:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:44:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:43 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:43.949 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-12T08:44:44.001 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:43 smithi114 ceph-mon[33573]: pgmap v95: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:43 smithi114 ceph-mon[33573]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:44:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:43 smithi114 ceph-mon[33573]: osdmap e73: 8 total, 8 up, 8 in 2024-04-12T08:44:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:43 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' 2024-04-12T08:44:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:43 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:44:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:43 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:44:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:43 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:44:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:43 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:45.060 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:44 smithi090 ceph-mon[40003]: osdmap e74: 8 total, 8 up, 8 in 2024-04-12T08:44:45.060 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:44 smithi090 ceph-mon[29100]: osdmap e74: 8 total, 8 up, 8 in 2024-04-12T08:44:45.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:44 smithi114 ceph-mon[33573]: osdmap e74: 8 total, 8 up, 8 in 2024-04-12T08:44:45.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:44:45 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:44:45.596 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:45.893 INFO:teuthology.orchestra.run.smithi090.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-04-12T08:44:46.017 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:45 smithi090 ceph-mon[40003]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:46.017 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:45 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3274888270' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:44:46.017 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:45 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:44:46.018 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:45 smithi090 ceph-mon[29100]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:46.018 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:45 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3274888270' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:44:46.018 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:45 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:44:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:45 smithi114 ceph-mon[33573]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:45 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3274888270' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:44:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:45 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:44:46.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:46 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:44:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:46 smithi090 ceph-mon[40003]: osdmap e75: 8 total, 8 up, 8 in 2024-04-12T08:44:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:46 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:44:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:46 smithi090 ceph-mon[29100]: osdmap e75: 8 total, 8 up, 8 in 2024-04-12T08:44:47.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:46 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:44:47.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:46 smithi114 ceph-mon[33573]: osdmap e75: 8 total, 8 up, 8 in 2024-04-12T08:44:48.094 DEBUG:teuthology.orchestra.run.smithi090:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-04-12T08:44:48.097 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-12T08:44:48.097 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-12T08:44:48.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:hints = 1 2024-04-12T08:44:48.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-12T08:44:48.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Object prefix: benchmark_data_smithi090_79937 2024-04-12T08:44:48.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:44:48.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 0 0 0 0 0 0 - 0 2024-04-12T08:44:48.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:47 smithi090 ceph-mon[29100]: pgmap v100: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:48.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:47 smithi090 ceph-mon[29100]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:44:48.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:47 smithi090 ceph-mon[29100]: Cluster is now healthy 2024-04-12T08:44:48.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:47 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:48.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:47 smithi090 ceph-mon[40003]: pgmap v100: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:48.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:47 smithi090 ceph-mon[40003]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:44:48.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:47 smithi090 ceph-mon[40003]: Cluster is now healthy 2024-04-12T08:44:48.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:47 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:47 smithi114 ceph-mon[33573]: pgmap v100: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:44:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:47 smithi114 ceph-mon[33573]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:44:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:47 smithi114 ceph-mon[33573]: Cluster is now healthy 2024-04-12T08:44:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:47 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:44:49.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 1 16 4407 4391 274.424 274.438 0.00273247 0.00363376 2024-04-12T08:44:49.244 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-12T08:44:49.748 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:50.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 2 16 8315 8299 259.326 244.25 0.00573778 0.00385042 2024-04-12T08:44:50.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:49 smithi090 ceph-mon[40003]: pgmap v101: 89 pgs: 89 active+clean; 130 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s wr, 344 op/s 2024-04-12T08:44:50.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:49 smithi090 ceph-mon[29100]: pgmap v101: 89 pgs: 89 active+clean; 130 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s wr, 344 op/s 2024-04-12T08:44:50.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:49 smithi114 ceph-mon[33573]: pgmap v101: 89 pgs: 89 active+clean; 130 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s wr, 344 op/s 2024-04-12T08:44:51.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 3 16 12146 12130 252.692 239.438 0.00366557 0.00395142 2024-04-12T08:44:52.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 4 16 15488 15472 241.733 208.875 0.00542102 0.00413142 2024-04-12T08:44:52.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:51 smithi090 ceph-mon[40003]: pgmap v102: 89 pgs: 89 active+clean; 130 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 266 op/s 2024-04-12T08:44:52.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:44:51 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:51] "GET /metrics HTTP/1.1" 200 241525 "" "Prometheus/2.43.0" 2024-04-12T08:44:52.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:51 smithi090 ceph-mon[29100]: pgmap v102: 89 pgs: 89 active+clean; 130 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 266 op/s 2024-04-12T08:44:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:51 smithi114 ceph-mon[33573]: pgmap v102: 89 pgs: 89 active+clean; 130 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 266 op/s 2024-04-12T08:44:52.482 INFO:teuthology.orchestra.run.smithi090.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:44:53.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 5 16 18320 18304 228.783 177 0.00593818 0.00436505 2024-04-12T08:44:53.481 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch ps' 2024-04-12T08:44:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:53 smithi114 ceph-mon[33573]: from='client.24595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:44:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:53 smithi114 ceph-mon[33573]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:44:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:53 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:53 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:44:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:53 smithi114 ceph-mon[33573]: pgmap v103: 89 pgs: 89 active+clean; 130 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 236 op/s 2024-04-12T08:44:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:53 smithi090 ceph-mon[29100]: from='client.24595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:44:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:53 smithi090 ceph-mon[29100]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:44:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:53 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:53 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:44:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:53 smithi090 ceph-mon[29100]: pgmap v103: 89 pgs: 89 active+clean; 130 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 236 op/s 2024-04-12T08:44:53.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:53 smithi090 ceph-mon[40003]: from='client.24595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:44:53.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:53 smithi090 ceph-mon[40003]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:44:53.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:53 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:53.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:53 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:44:53.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:53 smithi090 ceph-mon[40003]: pgmap v103: 89 pgs: 89 active+clean; 130 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 236 op/s 2024-04-12T08:44:54.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 6 16 21031 21015 218.89 169.438 0.00205031 0.00456193 2024-04-12T08:44:54.530 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:44:55.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 7 16 23786 23770 212.216 172.188 0.00347929 0.00470716 2024-04-12T08:44:55.283 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:44:55 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:44:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:44:55.878 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:55 smithi090 ceph-mon[40003]: pgmap v104: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-04-12T08:44:55.878 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:55 smithi090 ceph-mon[29100]: pgmap v104: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-04-12T08:44:56.002 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:55 smithi114 ceph-mon[33573]: pgmap v104: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-04-12T08:44:56.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 8 16 26505 26489 206.929 169.938 0.00528636 0.00482773 2024-04-12T08:44:57.013 INFO:teuthology.orchestra.run.smithi090.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T08:44:57.013 INFO:teuthology.orchestra.run.smithi090.stdout:alertmanager.a smithi090 *:9093,9094 running (97s) 70s ago 2m 22.6M - 0.25.0 c8568f914cd2 0893314a6971 2024-04-12T08:44:57.013 INFO:teuthology.orchestra.run.smithi090.stdout:grafana.a smithi114 *:3000 running (2m) 72s ago 2m 82.6M - 9.4.7 954c08fa6188 44058eaefb74 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi090.rxdrqj smithi090 running (84s) 70s ago 83s 15.8M - 17.2.7-827-gd3b5060a 21de9b7369ad f627bce7c272 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi114.nbzjob smithi114 running (81s) 72s ago 81s 11.8M - 17.2.7-827-gd3b5060a 21de9b7369ad 60cb65499580 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.x smithi114 *:8443 running (6m) 72s ago 6m 432M - 17.2.7-827-gd3b5060a 21de9b7369ad 1d7b1e033afe 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.y smithi090 *:9283 running (8m) 70s ago 8m 475M - 17.2.7-827-gd3b5060a 21de9b7369ad 7ed4ce011c21 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:mon.a smithi090 running (8m) 70s ago 8m 44.1M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 5c90fd1861be 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:mon.b smithi114 running (6m) 72s ago 6m 37.6M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 63cd6193074e 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:mon.c smithi090 running (6m) 70s ago 6m 38.3M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad d5aa934587fe 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.a smithi090 *:9100 running (2m) 70s ago 2m 16.1M - 1.5.0 0da6a335fe13 b71f7d0a7569 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.b smithi114 *:9100 running (2m) 72s ago 2m 15.9M - 1.5.0 0da6a335fe13 9a6215e32fcf 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:osd.0 smithi090 running (6m) 70s ago 6m 41.3M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 9395ed639a37 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:osd.1 smithi090 running (5m) 70s ago 5m 44.4M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 98eb8b5854e9 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:osd.2 smithi090 running (5m) 70s ago 5m 39.5M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 8008248b4e61 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:osd.3 smithi090 running (5m) 70s ago 5m 42.7M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 83a5641ba19a 2024-04-12T08:44:57.014 INFO:teuthology.orchestra.run.smithi090.stdout:osd.4 smithi114 running (4m) 72s ago 4m 40.2M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 15d0d8563392 2024-04-12T08:44:57.015 INFO:teuthology.orchestra.run.smithi090.stdout:osd.5 smithi114 running (4m) 72s ago 4m 42.8M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 54049e134882 2024-04-12T08:44:57.015 INFO:teuthology.orchestra.run.smithi090.stdout:osd.6 smithi114 running (3m) 72s ago 3m 41.3M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 86dfedc8629b 2024-04-12T08:44:57.015 INFO:teuthology.orchestra.run.smithi090.stdout:osd.7 smithi114 running (3m) 72s ago 3m 42.4M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad dda597b19712 2024-04-12T08:44:57.015 INFO:teuthology.orchestra.run.smithi090.stdout:prometheus.a smithi114 *:9095 running (92s) 72s ago 3m 32.5M - 2.43.0 a07b618ecd1d 94ac082be83b 2024-04-12T08:44:57.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 9 16 28838 28822 200.137 145.812 0.00239371 0.0049915 2024-04-12T08:44:58.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:57 smithi114 ceph-mon[33573]: pgmap v105: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-04-12T08:44:58.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:57 smithi114 ceph-mon[33573]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:44:58.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:57 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:58.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:57 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:58.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 10 15 31364 31349 195.916 157.938 0.00389249 0.00509912 2024-04-12T08:44:58.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:57 smithi090 ceph-mon[29100]: pgmap v105: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-04-12T08:44:58.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:57 smithi090 ceph-mon[29100]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:44:58.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:57 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:58.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:57 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:58.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:57 smithi090 ceph-mon[40003]: pgmap v105: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-04-12T08:44:58.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:57 smithi090 ceph-mon[40003]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:44:58.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:57 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:58.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:57 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:44:59.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 11 16 33307 33291 189.138 121.375 0.00713025 0.00528236 2024-04-12T08:45:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:44:59 smithi114 ceph-mon[33573]: pgmap v106: 89 pgs: 89 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s 2024-04-12T08:45:00.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 12 16 35254 35238 183.517 121.688 0.008178 0.00544412 2024-04-12T08:45:00.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:44:59 smithi090 ceph-mon[29100]: pgmap v106: 89 pgs: 89 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s 2024-04-12T08:45:00.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:44:59 smithi090 ceph-mon[40003]: pgmap v106: 89 pgs: 89 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s 2024-04-12T08:45:01.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 13 16 36842 36826 177.034 99.25 0.00564801 0.00564366 2024-04-12T08:45:02.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:01 smithi114 ceph-mon[33573]: pgmap v107: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-04-12T08:45:02.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 14 16 37880 37864 169.022 64.875 0.0100196 0.00591217 2024-04-12T08:45:02.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:01 smithi090 ceph-mon[29100]: pgmap v107: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-04-12T08:45:02.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:45:01 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:01] "GET /metrics HTTP/1.1" 200 241525 "" "Prometheus/2.43.0" 2024-04-12T08:45:02.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:01 smithi090 ceph-mon[40003]: pgmap v107: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-04-12T08:45:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:03.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 15 16 39626 39610 165.028 109.125 0.00721356 0.00605435 2024-04-12T08:45:03.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:03.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:03 smithi114 ceph-mon[33573]: pgmap v108: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-04-12T08:45:04.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 16 16 41265 41249 161.116 102.438 0.00805202 0.00614588 2024-04-12T08:45:04.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:03 smithi090 ceph-mon[29100]: pgmap v108: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-04-12T08:45:04.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:03 smithi090 ceph-mon[40003]: pgmap v108: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-04-12T08:45:05.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 17 16 42658 42642 156.759 87.0625 0.00773946 0.00637463 2024-04-12T08:45:05.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:45:05 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:45:05.781 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-12T08:45:06.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:05 smithi114 ceph-mon[33573]: pgmap v109: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 200 MiB/s wr, 3.20k op/s 2024-04-12T08:45:06.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 18 15 44778 44763 155.414 132.562 0.00961471 0.00642979 2024-04-12T08:45:06.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:05 smithi090 ceph-mon[29100]: pgmap v109: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 200 MiB/s wr, 3.20k op/s 2024-04-12T08:45:06.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:05 smithi090 ceph-mon[40003]: pgmap v109: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 200 MiB/s wr, 3.20k op/s 2024-04-12T08:45:06.815 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:45:07.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 19 16 46781 46765 153.82 125.125 0.00303956 0.00649286 2024-04-12T08:45:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:07 smithi114 ceph-mon[33573]: pgmap v110: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-12T08:45:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:07 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:45:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:07 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:45:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:07 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:45:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:07 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:45:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:07 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:45:08.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:07 smithi114 ceph-mon[33573]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:45:08.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:45:08.145960+0000 min lat: 0.00122027 max lat: 0.408439 avg lat: 0.00659195 2024-04-12T08:45:08.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:45:08.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 20 16 48526 48510 151.581 109.062 0.0054419 0.00659195 2024-04-12T08:45:08.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[29100]: pgmap v110: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[29100]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[40003]: pgmap v110: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:45:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:45:08.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:07 smithi090 ceph-mon[40003]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T08:45:09.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 21 16 50590 50574 150.506 129 0.0104361 0.00663961 2024-04-12T08:45:10.034 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:09 smithi090 ceph-mon[40003]: pgmap v111: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-04-12T08:45:10.034 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:09 smithi090 ceph-mon[29100]: pgmap v111: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-04-12T08:45:10.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:09 smithi114 ceph-mon[33573]: pgmap v111: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-04-12T08:45:10.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 22 16 52535 52519 149.19 121.562 0.00527443 0.00669839 2024-04-12T08:45:11.146 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 23 16 54307 54291 147.517 110.75 0.0118169 0.00677337 2024-04-12T08:45:11.240 INFO:teuthology.orchestra.run.smithi090.stdout:wait for minority of mons to upgrade 2024-04-12T08:45:12.065 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:45:11 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:11] "GET /metrics HTTP/1.1" 200 242542 "" "Prometheus/2.43.0" 2024-04-12T08:45:12.066 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:11 smithi090 ceph-mon[40003]: pgmap v112: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-04-12T08:45:12.066 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:11 smithi090 ceph-mon[29100]: pgmap v112: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-04-12T08:45:12.108 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-12T08:45:12.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:11 smithi114 ceph-mon[33573]: pgmap v112: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-04-12T08:45:12.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 24 16 56090 56074 146.009 111.438 0.00959856 0.00684409 2024-04-12T08:45:12.584 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:45:13.146 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 25 16 57784 57768 144.404 105.875 0.00383657 0.00692039 2024-04-12T08:45:14.112 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:13 smithi090 ceph-mon[40003]: pgmap v113: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-12T08:45:14.113 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:13 smithi090 ceph-mon[29100]: pgmap v113: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-12T08:45:14.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:13 smithi114 ceph-mon[33573]: pgmap v113: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-12T08:45:14.146 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 26 16 59533 59517 143.054 109.312 0.00841022 0.00698582 2024-04-12T08:45:15.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 27 16 61163 61147 141.528 101.875 0.0091781 0.00706106 2024-04-12T08:45:15.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:45:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:45:16.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:15 smithi114 ceph-mon[33573]: pgmap v114: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-04-12T08:45:16.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:15 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2314949791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:16.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 28 16 62829 62813 140.192 104.125 0.00563942 0.00712848 2024-04-12T08:45:16.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:15 smithi090 ceph-mon[40003]: pgmap v114: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-04-12T08:45:16.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:15 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2314949791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:16.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:15 smithi090 ceph-mon[29100]: pgmap v114: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-04-12T08:45:16.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:15 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2314949791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:17.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 29 16 64311 64295 138.552 92.625 0.00831849 0.00721286 2024-04-12T08:45:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:17 smithi114 ceph-mon[33573]: pgmap v115: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-12T08:45:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:17 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:18.119 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:17 smithi090 ceph-mon[29100]: pgmap v115: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-12T08:45:18.119 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:17 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:18.119 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:17 smithi090 ceph-mon[40003]: pgmap v115: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-12T08:45:18.120 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:17 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:18.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 30 16 65515 65499 136.441 75.25 0.0114989 0.0073242 2024-04-12T08:45:19.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 31 16 67079 67063 135.193 97.75 0.0062309 0.00739252 2024-04-12T08:45:19.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:18 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/176098471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:19.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:18 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/176098471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:19.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:18 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/176098471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:20.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 32 16 68655 68639 134.046 98.5 0.00962768 0.00745569 2024-04-12T08:45:20.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:19 smithi090 ceph-mon[40003]: pgmap v116: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-04-12T08:45:20.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:19 smithi090 ceph-mon[29100]: pgmap v116: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-04-12T08:45:20.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:19 smithi114 ceph-mon[33573]: pgmap v116: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-04-12T08:45:21.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 33 16 70119 70103 132.757 91.5 0.0113308 0.00752793 2024-04-12T08:45:21.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:20 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4246633992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:21.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:20 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4246633992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:21.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:20 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4246633992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:22.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 34 16 71648 71632 131.663 95.5625 0.0146091 0.00759081 2024-04-12T08:45:22.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:45:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:21] "GET /metrics HTTP/1.1" 200 242673 "" "Prometheus/2.43.0" 2024-04-12T08:45:22.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:21 smithi090 ceph-mon[40003]: pgmap v117: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-12T08:45:22.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:21 smithi090 ceph-mon[29100]: pgmap v117: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-12T08:45:22.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:21 smithi114 ceph-mon[33573]: pgmap v117: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-12T08:45:23.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 35 16 73108 73092 130.508 91.25 0.00859071 0.00765813 2024-04-12T08:45:24.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 36 16 74519 74503 129.332 88.1875 0.00548108 0.00772739 2024-04-12T08:45:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:23 smithi090 ceph-mon[29100]: pgmap v118: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T08:45:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:23 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3796439451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:24.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:23 smithi090 ceph-mon[40003]: pgmap v118: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T08:45:24.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:23 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3796439451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:23 smithi114 ceph-mon[33573]: pgmap v118: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T08:45:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:23 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3796439451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:25.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 37 16 75998 75982 128.335 92.4375 0.010806 0.00778744 2024-04-12T08:45:25.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:45:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:45:26.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 38 16 77383 77367 127.235 86.5625 0.022786 0.00785461 2024-04-12T08:45:26.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:25 smithi090 ceph-mon[29100]: pgmap v119: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-04-12T08:45:26.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:25 smithi090 ceph-mon[40003]: pgmap v119: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-04-12T08:45:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:25 smithi114 ceph-mon[33573]: pgmap v119: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-04-12T08:45:27.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 39 15 78730 78715 126.132 84.25 0.00764531 0.00792332 2024-04-12T08:45:27.239 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:26 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4113379989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:27.239 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:26 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4113379989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:27.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:26 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4113379989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:28.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:45:28.148772+0000 min lat: 0.00122027 max lat: 0.408439 avg lat: 0.00798081 2024-04-12T08:45:28.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:45:28.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 40 16 80170 80154 125.227 89.9375 0.00679887 0.00798081 2024-04-12T08:45:28.270 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:27 smithi090 ceph-mon[40003]: pgmap v120: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-12T08:45:28.270 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:27 smithi090 ceph-mon[29100]: pgmap v120: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-12T08:45:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:27 smithi114 ceph-mon[33573]: pgmap v120: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-12T08:45:29.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 41 16 81467 81451 124.149 81.0625 0.0142553 0.00805038 2024-04-12T08:45:30.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 42 16 82865 82849 123.274 87.375 0.0105832 0.0081073 2024-04-12T08:45:30.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:29 smithi114 ceph-mon[33573]: pgmap v121: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-12T08:45:30.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:29 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3020315770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:30.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:29 smithi090 ceph-mon[29100]: pgmap v121: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-12T08:45:30.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:29 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3020315770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:30.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:29 smithi090 ceph-mon[40003]: pgmap v121: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-12T08:45:30.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:29 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3020315770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:31.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 43 16 84246 84230 122.414 86.3125 0.0136858 0.00816413 2024-04-12T08:45:32.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 44 15 85536 85521 121.465 80.6875 0.0252599 0.00822791 2024-04-12T08:45:32.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:31 smithi090 ceph-mon[29100]: pgmap v122: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-12T08:45:32.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:31 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3155512748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:32.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:45:31 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:31] "GET /metrics HTTP/1.1" 200 242673 "" "Prometheus/2.43.0" 2024-04-12T08:45:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:31 smithi090 ceph-mon[40003]: pgmap v122: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-12T08:45:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:31 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3155512748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:31 smithi114 ceph-mon[33573]: pgmap v122: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-12T08:45:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:31 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3155512748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:33.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 45 16 86834 86818 120.567 81.0625 0.00955813 0.00828942 2024-04-12T08:45:33.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:32 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:33.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:32 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:33.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:32 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:34.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 46 16 88104 88088 119.672 79.375 0.00835762 0.00835167 2024-04-12T08:45:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:33 smithi114 ceph-mon[33573]: pgmap v123: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-12T08:45:34.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:33 smithi090 ceph-mon[29100]: pgmap v123: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-12T08:45:34.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:33 smithi090 ceph-mon[40003]: pgmap v123: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-12T08:45:35.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 47 16 89376 89360 118.817 79.5 0.00631842 0.0084112 2024-04-12T08:45:35.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:45:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:45:35.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:34 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3763431693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:35.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:34 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3763431693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:35.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:34 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3763431693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:36.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 48 16 90681 90665 118.041 81.5625 0.0112852 0.00846625 2024-04-12T08:45:36.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:35 smithi114 ceph-mon[33573]: pgmap v124: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-04-12T08:45:36.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:35 smithi090 ceph-mon[40003]: pgmap v124: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-04-12T08:45:36.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:35 smithi090 ceph-mon[29100]: pgmap v124: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-04-12T08:45:37.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 49 16 91883 91867 117.165 75.125 0.00920515 0.00852994 2024-04-12T08:45:38.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 50 16 93144 93128 116.398 78.8125 0.00860293 0.00858645 2024-04-12T08:45:38.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:37 smithi114 ceph-mon[33573]: pgmap v125: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T08:45:38.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:37 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3801110696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:38.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:37 smithi090 ceph-mon[40003]: pgmap v125: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T08:45:38.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:37 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3801110696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:38.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:37 smithi090 ceph-mon[29100]: pgmap v125: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T08:45:38.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:37 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3801110696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:39.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 51 16 94414 94398 115.672 79.375 0.00891429 0.00864053 2024-04-12T08:45:40.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 52 16 95546 95530 114.808 70.75 0.00847823 0.0087056 2024-04-12T08:45:40.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:39 smithi114 ceph-mon[33573]: pgmap v126: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-12T08:45:40.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:39 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1116514307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:40.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:39 smithi090 ceph-mon[29100]: pgmap v126: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-12T08:45:40.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:39 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1116514307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:40.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:39 smithi090 ceph-mon[40003]: pgmap v126: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-12T08:45:40.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:39 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1116514307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:41.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 53 16 96773 96757 114.088 76.6875 0.00786249 0.00876056 2024-04-12T08:45:42.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 54 16 97959 97943 113.348 74.125 0.0127336 0.00881768 2024-04-12T08:45:42.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:41 smithi090 ceph-mon[29100]: pgmap v127: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-04-12T08:45:42.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:45:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:41] "GET /metrics HTTP/1.1" 200 242741 "" "Prometheus/2.43.0" 2024-04-12T08:45:42.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:41 smithi090 ceph-mon[40003]: pgmap v127: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-04-12T08:45:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:41 smithi114 ceph-mon[33573]: pgmap v127: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-04-12T08:45:43.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 55 16 99084 99068 112.565 70.3125 0.00790421 0.00887819 2024-04-12T08:45:43.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:42 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/927570406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:43.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:42 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/927570406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:43.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:42 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/927570406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:44.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 56 16 100269 100253 111.878 74.0625 0.0170261 0.00893385 2024-04-12T08:45:44.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:43 smithi114 ceph-mon[33573]: pgmap v128: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-04-12T08:45:44.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:43 smithi090 ceph-mon[40003]: pgmap v128: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-04-12T08:45:44.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:43 smithi090 ceph-mon[29100]: pgmap v128: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-04-12T08:45:45.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 57 16 101402 101386 111.157 70.8125 0.0128374 0.00899171 2024-04-12T08:45:45.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:45:45 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:45:46.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 58 16 102523 102507 110.449 70.0625 0.0096788 0.00904952 2024-04-12T08:45:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:45 smithi114 ceph-mon[33573]: pgmap v129: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-12T08:45:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:45 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2855500220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:46.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:45 smithi090 ceph-mon[29100]: pgmap v129: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-12T08:45:46.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:45 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2855500220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:46.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:46 smithi090 ceph-mon[40003]: pgmap v129: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-12T08:45:46.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:46 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2855500220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:47.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 59 16 103663 103647 109.784 71.25 0.00754287 0.00910358 2024-04-12T08:45:48.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:45:48.150529+0000 min lat: 0.00122027 max lat: 0.408439 avg lat: 0.00918519 2024-04-12T08:45:48.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:45:48.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 60 16 104475 104459 108.8 50.75 0.0156824 0.00918519 2024-04-12T08:45:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:48 smithi114 ceph-mon[33573]: pgmap v130: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-04-12T08:45:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:48 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:48.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:48 smithi090 ceph-mon[40003]: pgmap v130: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-04-12T08:45:48.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:48 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:48.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:48 smithi090 ceph-mon[29100]: pgmap v130: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-04-12T08:45:48.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:48 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:45:49.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 61 16 105412 105396 107.974 58.5625 0.0124777 0.00925645 2024-04-12T08:45:49.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:49 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2247864320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:49 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2247864320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:49.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:49 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2247864320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:50.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 62 16 106466 106450 107.292 65.875 0.025401 0.00931524 2024-04-12T08:45:50.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:50 smithi114 ceph-mon[33573]: pgmap v131: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-12T08:45:50.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:50 smithi090 ceph-mon[29100]: pgmap v131: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-12T08:45:50.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:50 smithi090 ceph-mon[40003]: pgmap v131: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-12T08:45:51.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 63 16 107412 107396 106.527 59.125 0.0132969 0.00938186 2024-04-12T08:45:51.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:51 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1962203617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:51.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:51 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1962203617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:51.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:51 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1962203617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:52.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 64 16 108452 108436 105.878 65 0.0235089 0.00944001 2024-04-12T08:45:52.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:45:51 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:51] "GET /metrics HTTP/1.1" 200 242764 "" "Prometheus/2.43.0" 2024-04-12T08:45:52.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:52 smithi090 ceph-mon[40003]: pgmap v132: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-12T08:45:52.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:52 smithi090 ceph-mon[29100]: pgmap v132: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-12T08:45:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:52 smithi114 ceph-mon[33573]: pgmap v132: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-12T08:45:53.154 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 65 16 109416 109400 105.176 60.25 0.0320251 0.00950243 2024-04-12T08:45:54.154 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 66 16 110369 110353 104.485 59.5625 0.00815917 0.00956532 2024-04-12T08:45:54.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:54 smithi114 ceph-mon[33573]: pgmap v133: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-04-12T08:45:54.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:54 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1463026711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:54.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:54 smithi090 ceph-mon[29100]: pgmap v133: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-04-12T08:45:54.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:54 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1463026711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:54.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:54 smithi090 ceph-mon[40003]: pgmap v133: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-04-12T08:45:54.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:54 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1463026711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:55.154 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 67 16 111330 111314 103.822 60.0625 0.0137282 0.00962692 2024-04-12T08:45:55.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:45:55 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:45:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:45:56.154 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 68 16 112284 112268 103.171 59.625 0.0139725 0.00968715 2024-04-12T08:45:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:56 smithi114 ceph-mon[33573]: pgmap v134: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:45:56.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:56 smithi090 ceph-mon[29100]: pgmap v134: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:45:56.425 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:56 smithi090 ceph-mon[40003]: pgmap v134: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:45:57.155 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 69 16 113268 113252 102.566 61.5 0.0124512 0.00974496 2024-04-12T08:45:57.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:57 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3569927153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:57.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:57 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3569927153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:57.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:57 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3569927153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:58.155 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 70 16 114262 114246 101.989 62.125 0.0217212 0.00980013 2024-04-12T08:45:58.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:58 smithi114 ceph-mon[33573]: pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2024-04-12T08:45:58.407 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:58 smithi090 ceph-mon[40003]: pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2024-04-12T08:45:58.407 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:58 smithi090 ceph-mon[29100]: pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2024-04-12T08:45:59.158 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 71 16 115307 115291 101.468 65.3125 0.0160356 0.00985036 2024-04-12T08:45:59.341 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:45:59 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/38274645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:59.341 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:45:59 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/38274645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:45:59.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:45:59 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/38274645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:00.158 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 72 16 116380 116364 100.99 67.0625 0.00829736 0.00989715 2024-04-12T08:46:00.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:00 smithi114 ceph-mon[33573]: pgmap v136: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-04-12T08:46:00.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:00 smithi090 ceph-mon[40003]: pgmap v136: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-04-12T08:46:00.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:00 smithi090 ceph-mon[29100]: pgmap v136: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-04-12T08:46:01.158 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 73 16 117345 117329 100.433 60.3125 0.0144942 0.00995215 2024-04-12T08:46:02.090 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:46:01 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:01] "GET /metrics HTTP/1.1" 200 242764 "" "Prometheus/2.43.0" 2024-04-12T08:46:02.090 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:02 smithi090 ceph-mon[29100]: pgmap v137: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-12T08:46:02.090 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:02 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3773985988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:02.158 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 74 16 118236 118220 99.828 55.6875 0.0157194 0.0100123 2024-04-12T08:46:02.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:02 smithi114 ceph-mon[33573]: pgmap v137: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-12T08:46:02.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:02 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3773985988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:02.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:02 smithi090 ceph-mon[40003]: pgmap v137: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-12T08:46:02.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:02 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3773985988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:03.158 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 75 16 119026 119010 99.1553 49.375 0.0210204 0.0100801 2024-04-12T08:46:03.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:03 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:03.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:03 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:03.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:03 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:04.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 76 16 120052 120036 98.6944 64.125 0.0125901 0.0101279 2024-04-12T08:46:04.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:04 smithi114 ceph-mon[33573]: pgmap v138: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-04-12T08:46:04.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:04 smithi090 ceph-mon[40003]: pgmap v138: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-04-12T08:46:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:04 smithi090 ceph-mon[29100]: pgmap v138: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-04-12T08:46:05.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 77 16 121031 121015 98.2073 61.1875 0.0199629 0.0101776 2024-04-12T08:46:05.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:05 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3900777865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:05.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:46:05 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:46:05.402 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:05 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3900777865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:05.402 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:05 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3900777865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:06.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 78 16 121958 121942 97.691 57.9375 0.0216331 0.0102311 2024-04-12T08:46:06.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:06 smithi090 ceph-mon[40003]: pgmap v139: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-12T08:46:06.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:46:06.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:46:06.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:06 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:46:06.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:06 smithi090 ceph-mon[29100]: pgmap v139: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-12T08:46:06.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:46:06.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:46:06.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:06 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:46:06.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:06 smithi114 ceph-mon[33573]: pgmap v139: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-12T08:46:06.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:46:06.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:46:06.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:06 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:46:07.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 79 16 122887 122871 97.1893 58.0625 0.0289737 0.0102842 2024-04-12T08:46:07.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:07 smithi090 ceph-mon[29100]: Upgrade: Target is version 19.0.0-2053-gb277feb2 (unknown) 2024-04-12T08:46:07.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:07 smithi090 ceph-mon[29100]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec'] 2024-04-12T08:46:07.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:07 smithi090 ceph-mon[29100]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-12T08:46:07.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:07 smithi090 ceph-mon[29100]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec on smithi114 2024-04-12T08:46:07.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:07 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/764213805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:07 smithi090 ceph-mon[40003]: Upgrade: Target is version 19.0.0-2053-gb277feb2 (unknown) 2024-04-12T08:46:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:07 smithi090 ceph-mon[40003]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec'] 2024-04-12T08:46:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:07 smithi090 ceph-mon[40003]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-12T08:46:07.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:07 smithi090 ceph-mon[40003]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec on smithi114 2024-04-12T08:46:07.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:07 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/764213805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:07.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:07 smithi114 ceph-mon[33573]: Upgrade: Target is version 19.0.0-2053-gb277feb2 (unknown) 2024-04-12T08:46:07.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:07 smithi114 ceph-mon[33573]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec'] 2024-04-12T08:46:07.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:07 smithi114 ceph-mon[33573]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-12T08:46:07.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:07 smithi114 ceph-mon[33573]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec on smithi114 2024-04-12T08:46:07.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:07 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/764213805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:08.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:46:08.159631+0000 min lat: 0.00122027 max lat: 0.408439 avg lat: 0.0103356 2024-04-12T08:46:08.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:46:08.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 80 16 123824 123808 96.7065 58.5625 0.0145029 0.0103356 2024-04-12T08:46:08.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:08 smithi090 ceph-mon[29100]: pgmap v140: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2024-04-12T08:46:08.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:08 smithi090 ceph-mon[40003]: pgmap v140: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2024-04-12T08:46:08.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:08 smithi114 ceph-mon[33573]: pgmap v140: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2024-04-12T08:46:09.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 81 16 124723 124707 96.2062 56.1875 0.0246798 0.0103892 2024-04-12T08:46:10.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 82 16 125634 125618 95.7272 56.9375 0.00844338 0.0104415 2024-04-12T08:46:10.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:10 smithi090 ceph-mon[29100]: pgmap v141: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2024-04-12T08:46:10.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:10 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1190141769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:10.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:10 smithi090 ceph-mon[40003]: pgmap v141: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2024-04-12T08:46:10.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:10 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1190141769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:10.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:10 smithi114 ceph-mon[33573]: pgmap v141: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2024-04-12T08:46:10.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:10 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1190141769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:11.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 83 16 126589 126573 95.293 59.6875 0.0106169 0.0104889 2024-04-12T08:46:12.142 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:46:11 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:11] "GET /metrics HTTP/1.1" 200 242790 "" "Prometheus/2.43.0" 2024-04-12T08:46:12.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 84 16 127511 127495 94.8445 57.625 0.0117743 0.010539 2024-04-12T08:46:12.418 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:12 smithi114 ceph-mon[33573]: pgmap v142: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:46:12.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:12 smithi090 ceph-mon[40003]: pgmap v142: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:46:12.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:12 smithi090 ceph-mon[29100]: pgmap v142: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:46:13.159 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 85 16 128430 128414 94.4044 57.4375 0.0111804 0.0105875 2024-04-12T08:46:13.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:13 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3525756744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:13.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:13 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3525756744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:13.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:13 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3525756744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:14.160 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 86 16 129357 129341 93.9804 57.9375 0.016 0.010636 2024-04-12T08:46:14.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:14 smithi090 ceph-mon[40003]: pgmap v143: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-04-12T08:46:14.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:14 smithi090 ceph-mon[29100]: pgmap v143: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-04-12T08:46:14.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:14 smithi114 ceph-mon[33573]: pgmap v143: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-04-12T08:46:15.160 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 87 16 130262 130246 93.5503 56.5625 0.0211327 0.0106847 2024-04-12T08:46:15.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:46:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:46:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:15 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/245373696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:15.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:15 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/245373696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:15.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:15 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/245373696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:16.160 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 88 16 131173 131157 93.1342 56.9375 0.0168907 0.0107323 2024-04-12T08:46:16.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:16 smithi090 ceph-mon[40003]: pgmap v144: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-12T08:46:16.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:16 smithi090 ceph-mon[29100]: pgmap v144: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-12T08:46:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:16 smithi114 ceph-mon[33573]: pgmap v144: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-12T08:46:17.160 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 89 16 132097 132081 92.7366 57.75 0.00900427 0.0107785 2024-04-12T08:46:18.160 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 90 2 132948 132946 92.3069 54.0625 0.0228407 0.0108287 2024-04-12T08:46:18.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:18 smithi114 ceph-mon[33573]: pgmap v145: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2024-04-12T08:46:18.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:18 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/774514291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:18.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:18 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:18.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:18 smithi090 ceph-mon[40003]: pgmap v145: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2024-04-12T08:46:18.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:18 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/774514291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:18.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:18 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:18.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:18 smithi090 ceph-mon[29100]: pgmap v145: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2024-04-12T08:46:18.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:18 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/774514291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:18.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:18 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:19.160 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Total time run: 90.0179 2024-04-12T08:46:19.160 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Total writes made: 132948 2024-04-12T08:46:19.160 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Write size: 65536 2024-04-12T08:46:19.160 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Object size: 65536 2024-04-12T08:46:19.161 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Bandwidth (MB/sec): 92.3067 2024-04-12T08:46:19.161 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev Bandwidth: 44.1787 2024-04-12T08:46:19.161 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max bandwidth (MB/sec): 274.438 2024-04-12T08:46:19.161 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min bandwidth (MB/sec): 49.375 2024-04-12T08:46:19.161 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Average IOPS: 1476 2024-04-12T08:46:19.161 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev IOPS: 706.86 2024-04-12T08:46:19.161 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max IOPS: 4391 2024-04-12T08:46:19.161 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min IOPS: 790 2024-04-12T08:46:19.161 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Average Latency(s): 0.0108289 2024-04-12T08:46:19.161 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev Latency(s): 0.0101342 2024-04-12T08:46:19.162 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max latency(s): 0.408439 2024-04-12T08:46:19.162 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min latency(s): 0.00122027 2024-04-12T08:46:19.166 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Cleaning up (deleting benchmark objects) 2024-04-12T08:46:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:20 smithi114 ceph-mon[33573]: pgmap v146: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2024-04-12T08:46:20.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:20 smithi090 ceph-mon[40003]: pgmap v146: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2024-04-12T08:46:20.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:20 smithi090 ceph-mon[29100]: pgmap v146: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2024-04-12T08:46:21.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:21 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3731118967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:21.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:21 smithi114 ceph-mon[33573]: pgmap v147: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-12T08:46:21.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:21 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3731118967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:21.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:21 smithi090 ceph-mon[40003]: pgmap v147: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-12T08:46:21.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:21 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3731118967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:21.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:21 smithi090 ceph-mon[29100]: pgmap v147: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-12T08:46:22.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:46:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:21] "GET /metrics HTTP/1.1" 200 242814 "" "Prometheus/2.43.0" 2024-04-12T08:46:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:23 smithi114 ceph-mon[33573]: pgmap v148: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s 2024-04-12T08:46:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:23 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3797703360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:24.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:23 smithi090 ceph-mon[40003]: pgmap v148: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s 2024-04-12T08:46:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:23 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3797703360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:23 smithi090 ceph-mon[29100]: pgmap v148: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s 2024-04-12T08:46:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:23 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3797703360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:25.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:46:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:46:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:25 smithi114 ceph-mon[33573]: pgmap v149: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 1.07k op/s 2024-04-12T08:46:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:25 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3525071041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:26.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:25 smithi090 ceph-mon[40003]: pgmap v149: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 1.07k op/s 2024-04-12T08:46:26.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:25 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3525071041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:25 smithi090 ceph-mon[29100]: pgmap v149: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 1.07k op/s 2024-04-12T08:46:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:25 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3525071041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:28.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:27 smithi114 ceph-mon[33573]: pgmap v150: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 687 op/s 2024-04-12T08:46:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:27 smithi090 ceph-mon[29100]: pgmap v150: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 687 op/s 2024-04-12T08:46:28.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:27 smithi090 ceph-mon[40003]: pgmap v150: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 687 op/s 2024-04-12T08:46:29.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:28 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2518693911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:28 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2518693911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:29.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:28 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2518693911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:29 smithi114 ceph-mon[33573]: pgmap v151: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 687 op/s 2024-04-12T08:46:30.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:29 smithi090 ceph-mon[40003]: pgmap v151: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 687 op/s 2024-04-12T08:46:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:29 smithi090 ceph-mon[29100]: pgmap v151: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 687 op/s 2024-04-12T08:46:32.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:31 smithi114 ceph-mon[33573]: pgmap v152: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.07k op/s 2024-04-12T08:46:32.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:31 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/169293114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:32.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:46:31 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:31] "GET /metrics HTTP/1.1" 200 242814 "" "Prometheus/2.43.0" 2024-04-12T08:46:32.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:31 smithi090 ceph-mon[29100]: pgmap v152: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.07k op/s 2024-04-12T08:46:32.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:31 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/169293114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:31 smithi090 ceph-mon[40003]: pgmap v152: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.07k op/s 2024-04-12T08:46:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:31 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/169293114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:33.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:32 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:33.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:32 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:33.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:32 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:34.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:33 smithi114 ceph-mon[33573]: pgmap v153: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 734 op/s 2024-04-12T08:46:34.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:33 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/945954573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:34.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:33 smithi090 ceph-mon[40003]: pgmap v153: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 734 op/s 2024-04-12T08:46:34.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:33 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/945954573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:34.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:33 smithi090 ceph-mon[29100]: pgmap v153: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 734 op/s 2024-04-12T08:46:34.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:33 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/945954573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:35.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:46:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:46:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:35 smithi114 ceph-mon[33573]: pgmap v154: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 1.10k op/s 2024-04-12T08:46:36.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:35 smithi090 ceph-mon[29100]: pgmap v154: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 1.10k op/s 2024-04-12T08:46:36.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:35 smithi090 ceph-mon[40003]: pgmap v154: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 1.10k op/s 2024-04-12T08:46:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:36 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3602219608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:37.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:36 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3602219608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:37.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:36 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3602219608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:37 smithi114 ceph-mon[33573]: pgmap v155: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-04-12T08:46:38.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:37 smithi090 ceph-mon[29100]: pgmap v155: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-04-12T08:46:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:37 smithi090 ceph-mon[40003]: pgmap v155: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-04-12T08:46:39.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:38 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/404439245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:39.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:38 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/404439245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:39.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:38 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/404439245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:40.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:39 smithi114 ceph-mon[33573]: pgmap v156: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-04-12T08:46:40.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:39 smithi090 ceph-mon[40003]: pgmap v156: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-04-12T08:46:40.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:39 smithi090 ceph-mon[29100]: pgmap v156: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-04-12T08:46:42.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:41 smithi114 ceph-mon[33573]: pgmap v157: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-12T08:46:42.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:41 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3525310247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:42.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:46:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:41] "GET /metrics HTTP/1.1" 200 242898 "" "Prometheus/2.43.0" 2024-04-12T08:46:42.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:41 smithi090 ceph-mon[40003]: pgmap v157: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-12T08:46:42.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:41 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3525310247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:42.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:41 smithi090 ceph-mon[29100]: pgmap v157: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-12T08:46:42.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:41 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3525310247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:43 smithi114 ceph-mon[33573]: pgmap v158: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-04-12T08:46:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:43 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/158267830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:44.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:43 smithi090 ceph-mon[40003]: pgmap v158: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-04-12T08:46:44.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:43 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/158267830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:43 smithi090 ceph-mon[29100]: pgmap v158: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-04-12T08:46:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:43 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/158267830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:45.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:46:45 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:46:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:45 smithi114 ceph-mon[33573]: pgmap v159: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:46:46.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:45 smithi090 ceph-mon[29100]: pgmap v159: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:46:46.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:45 smithi090 ceph-mon[40003]: pgmap v159: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:46:47.074 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:46 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4275898254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:47.074 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:46 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4275898254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:47.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:46 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4275898254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:47 smithi114 ceph-mon[33573]: pgmap v160: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-04-12T08:46:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:47 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:48.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:47 smithi090 ceph-mon[40003]: pgmap v160: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-04-12T08:46:48.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:47 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:48.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:47 smithi090 ceph-mon[29100]: pgmap v160: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-04-12T08:46:48.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:47 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:46:50.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:49 smithi114 ceph-mon[33573]: pgmap v161: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-04-12T08:46:50.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:49 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2058025403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:50.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:49 smithi090 ceph-mon[40003]: pgmap v161: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-04-12T08:46:50.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:49 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2058025403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:50.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:49 smithi090 ceph-mon[29100]: pgmap v161: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-04-12T08:46:50.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:49 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2058025403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:51 smithi114 ceph-mon[33573]: pgmap v162: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:46:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:51 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2067414455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:52.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:46:51 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:51] "GET /metrics HTTP/1.1" 200 242901 "" "Prometheus/2.43.0" 2024-04-12T08:46:52.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:51 smithi090 ceph-mon[40003]: pgmap v162: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:46:52.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:51 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2067414455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:52.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:51 smithi090 ceph-mon[29100]: pgmap v162: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:46:52.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:51 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2067414455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:54.069 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:53 smithi114 ceph-mon[33573]: pgmap v163: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-04-12T08:46:54.070 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:53 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:46:54.070 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:53 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:46:54.070 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:53 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:46:54.070 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:53 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:46:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:53 smithi090 ceph-mon[40003]: pgmap v163: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-04-12T08:46:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:53 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:46:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:53 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:46:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:53 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:46:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:53 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:46:54.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:53 smithi090 ceph-mon[29100]: pgmap v163: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-04-12T08:46:54.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:53 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:46:54.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:53 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:46:54.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:53 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:46:54.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:53 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:46:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:54 smithi114 ceph-mon[33573]: Upgrade: Updating mgr.x 2024-04-12T08:46:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:54 smithi114 ceph-mon[33573]: Deploying daemon mgr.x on smithi114 2024-04-12T08:46:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:54 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2035605344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:55.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:54 smithi090 ceph-mon[40003]: Upgrade: Updating mgr.x 2024-04-12T08:46:55.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:54 smithi090 ceph-mon[40003]: Deploying daemon mgr.x on smithi114 2024-04-12T08:46:55.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:54 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2035605344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:55.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:54 smithi090 ceph-mon[29100]: Upgrade: Updating mgr.x 2024-04-12T08:46:55.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:54 smithi090 ceph-mon[29100]: Deploying daemon mgr.x on smithi114 2024-04-12T08:46:55.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:54 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2035605344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:55.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:46:55 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[35308]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:46:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:46:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:55 smithi114 ceph-mon[33573]: pgmap v164: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:46:56.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:55 smithi090 ceph-mon[40003]: pgmap v164: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:46:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:55 smithi090 ceph-mon[29100]: pgmap v164: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:46:57.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:46:56 smithi114 systemd[1]: Stopping Ceph mgr.x for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:46:58.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:57 smithi114 ceph-mon[33573]: pgmap v165: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-04-12T08:46:58.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:57 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/788011485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:58.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:57 smithi090 ceph-mon[40003]: pgmap v165: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-04-12T08:46:58.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:57 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/788011485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:58.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:57 smithi090 ceph-mon[29100]: pgmap v165: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-04-12T08:46:58.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:57 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/788011485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:46:59.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:46:58 smithi114 podman[66784]: 2024-04-12 08:46:58.859845594 +0000 UTC m=+1.573763180 container died 1d7b1e033afe78e6763e60ad450a8928704b90baf68e74227d1839f415ed9e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-12T08:47:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:59 smithi114 ceph-mon[33573]: pgmap v166: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-04-12T08:47:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:46:59 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/895195716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:00.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:59 smithi090 ceph-mon[40003]: pgmap v166: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-04-12T08:47:00.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:46:59 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/895195716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:00.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:59 smithi090 ceph-mon[29100]: pgmap v166: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-04-12T08:47:00.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:46:59 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/895195716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:02.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:01 smithi114 ceph-mon[33573]: pgmap v167: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:47:02.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:01 smithi090 ceph-mon[40003]: pgmap v167: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:47:02.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:01 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:47:01] "GET /metrics HTTP/1.1" 200 242901 "" "Prometheus/2.43.0" 2024-04-12T08:47:02.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:01 smithi090 ceph-mon[29100]: pgmap v167: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:47:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:02 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/53945534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:02 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:03.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:02 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/53945534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:03.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:02 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:03.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:02 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/53945534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:03.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:02 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:03 smithi114 ceph-mon[33573]: pgmap v168: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-04-12T08:47:04.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:03 smithi114 podman[66784]: 2024-04-12 08:47:03.782665723 +0000 UTC m=+6.496583303 container cleanup 1d7b1e033afe78e6763e60ad450a8928704b90baf68e74227d1839f415ed9e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, CEPH_POINT_RELEASE=) 2024-04-12T08:47:04.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:03 smithi114 bash[66784]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x 2024-04-12T08:47:04.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:03 smithi090 ceph-mon[40003]: pgmap v168: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-04-12T08:47:04.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:03 smithi090 ceph-mon[29100]: pgmap v168: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-04-12T08:47:06.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:06 smithi114 podman[66795]: 2024-04-12 08:47:06.071408284 +0000 UTC m=+7.209500606 container remove 1d7b1e033afe78e6763e60ad450a8928704b90baf68e74227d1839f415ed9e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-d3b5060, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-12T08:47:06.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:06 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-12T08:47:06.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:05 smithi114 ceph-mon[33573]: pgmap v169: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:47:06.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:05 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2527623860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:06.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:05 smithi090 ceph-mon[40003]: pgmap v169: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:47:06.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:05 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2527623860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:06.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:05 smithi090 ceph-mon[29100]: pgmap v169: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:47:06.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:05 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2527623860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:06.800 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:06 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-12T08:47:06.800 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:06 smithi114 systemd[1]: Stopped Ceph mgr.x for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:47:06.800 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:06 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.x.service: Consumed 18.606s CPU time. 2024-04-12T08:47:07.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:06 smithi114 systemd[1]: Starting Ceph mgr.x for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:47:07.444 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:07 smithi114 podman[66908]: 2024-04-12 08:47:07.027785538 +0000 UTC m=+0.043663973 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:47:07.444 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:07 smithi114 podman[66908]: 2024-04-12 08:47:07.156253957 +0000 UTC m=+0.172132389 container create 8e82f0405773258ab496cfaaf2e8b3893ac4ddb81e4c2c3610dff2f26f6ce37f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:47:07.865 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:07 smithi114 podman[66908]: 2024-04-12 08:47:07.44315214 +0000 UTC m=+0.459030580 container init 8e82f0405773258ab496cfaaf2e8b3893ac4ddb81e4c2c3610dff2f26f6ce37f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:47:07.865 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:07 smithi114 podman[66908]: 2024-04-12 08:47:07.448046532 +0000 UTC m=+0.463924976 container start 8e82f0405773258ab496cfaaf2e8b3893ac4ddb81e4c2c3610dff2f26f6ce37f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:47:07.865 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:07 smithi114 bash[66908]: 8e82f0405773258ab496cfaaf2e8b3893ac4ddb81e4c2c3610dff2f26f6ce37f 2024-04-12T08:47:07.865 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:07 smithi114 systemd[1]: Started Ceph mgr.x for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:47:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:07 smithi114 ceph-mon[33573]: pgmap v170: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-04-12T08:47:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:07 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1708502165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:07 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:07 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:07 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:47:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:07 smithi090 ceph-mon[40003]: pgmap v170: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-04-12T08:47:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:07 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1708502165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:07 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:07 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:07 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:47:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:07 smithi090 ceph-mon[29100]: pgmap v170: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-04-12T08:47:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:07 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1708502165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:07 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:07 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:07 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:47:08.783 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:08.402+0000 7f702ada6200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:47:08.783 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:08.570+0000 7f702ada6200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:47:09.805 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:09.632+0000 7f702ada6200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:47:10.106 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:09.802+0000 7f702ada6200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:47:10.106 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:10 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:10.029+0000 7f702ada6200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:47:10.106 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:09 smithi114 ceph-mon[33573]: pgmap v171: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-04-12T08:47:10.159 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:09 smithi090 ceph-mon[40003]: pgmap v171: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-04-12T08:47:10.159 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:09 smithi090 ceph-mon[29100]: pgmap v171: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-04-12T08:47:10.362 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:10 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:10.206+0000 7f702ada6200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:47:10.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:10 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:10.360+0000 7f702ada6200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:47:11.007 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:10 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/739373035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:11.008 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:10 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:10.816+0000 7f702ada6200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:47:11.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:10 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/739373035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:11.041 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:10 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/739373035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:11.328 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:11 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:11.006+0000 7f702ada6200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:47:11.328 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:11 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:11.166+0000 7f702ada6200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:47:11.603 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:11 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:11.326+0000 7f702ada6200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:47:11.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:11 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:11.602+0000 7f702ada6200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:47:12.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:11 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:47:11] "GET /metrics HTTP/1.1" 200 242946 "" "Prometheus/2.43.0" 2024-04-12T08:47:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:11 smithi090 ceph-mon[40003]: pgmap v172: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-12T08:47:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:11 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:11 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:11 smithi090 ceph-mon[29100]: pgmap v172: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-12T08:47:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:11 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:11 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:12.237 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:11 smithi114 ceph-mon[33573]: pgmap v172: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-12T08:47:12.237 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:11 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:12.238 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:11 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:13.140 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:12 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1128140821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:13.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:12 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1128140821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:13.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:12 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1128140821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:13.446 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:13 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:13.224+0000 7f702ada6200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:47:13.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:13 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:13.444+0000 7f702ada6200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:47:14.154 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:13 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:13.992+0000 7f702ada6200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:47:14.155 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:13 smithi114 ceph-mon[33573]: pgmap v173: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-04-12T08:47:14.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:13 smithi090 ceph-mon[40003]: pgmap v173: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-04-12T08:47:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:13 smithi090 ceph-mon[29100]: pgmap v173: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-04-12T08:47:14.496 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:14 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:14.153+0000 7f702ada6200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:47:14.822 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:14 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:14.494+0000 7f702ada6200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:47:15.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:14 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:14.819+0000 7f702ada6200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:47:15.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:14 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:15.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:14 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:15.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:14 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:15.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:14 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:15.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:14 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:15.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:14 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:15.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:15.321+0000 7f702ada6200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:47:15.983 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:15.718+0000 7f702ada6200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:47:16.238 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:16 smithi114 ceph-mon[33573]: pgmap v174: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 994 op/s 2024-04-12T08:47:16.239 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:16 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:16.239 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:16 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:16.239 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:16 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3945729940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:16.239 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:16 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:16.130+0000 7f702ada6200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:47:16.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:16 smithi090 ceph-mon[40003]: pgmap v174: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 994 op/s 2024-04-12T08:47:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:16 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:16 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:16 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3945729940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:16.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:16 smithi090 ceph-mon[29100]: pgmap v174: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 994 op/s 2024-04-12T08:47:16.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:16 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:16.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:16 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:16.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:16 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3945729940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:17.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:17 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:17.513+0000 7f702ada6200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:47:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:18 smithi114 ceph-mon[33573]: pgmap v175: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2024-04-12T08:47:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:18 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:18 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/776821120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:18.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:18 smithi090 ceph-mon[40003]: pgmap v175: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2024-04-12T08:47:18.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:18 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:18.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:18 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/776821120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:18 smithi090 ceph-mon[29100]: pgmap v175: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2024-04-12T08:47:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:18 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:18 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/776821120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:18.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:18.457+0000 7f702ada6200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:47:18.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:18.625+0000 7f702ada6200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:47:19.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:19.003+0000 7f702ada6200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:47:19.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:47:19] ENGINE Bus STARTING 2024-04-12T08:47:19.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: CherryPy Checker: 2024-04-12T08:47:19.368 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: The Application mounted at '' has an empty config. 2024-04-12T08:47:19.368 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:19.368 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:47:19] ENGINE Serving on http://:::9283 2024-04-12T08:47:19.368 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:47:19] ENGINE Bus STARTED 2024-04-12T08:47:19.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-mon[33573]: Standby manager daemon x restarted 2024-04-12T08:47:19.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-mon[33573]: Standby manager daemon x started 2024-04-12T08:47:19.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:47:19.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:47:19.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:47:19.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:19 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:47:19.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[40003]: Standby manager daemon x restarted 2024-04-12T08:47:19.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[40003]: Standby manager daemon x started 2024-04-12T08:47:19.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:47:19.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:47:19.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:47:19.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:47:19.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[29100]: Standby manager daemon x restarted 2024-04-12T08:47:19.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[29100]: Standby manager daemon x started 2024-04-12T08:47:19.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:47:19.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:47:19.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:47:19.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:19 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:47:20.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:20 smithi090 ceph-mon[40003]: pgmap v176: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2024-04-12T08:47:20.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:20 smithi090 ceph-mon[40003]: mgrmap e21: y(active, since 5m), standbys: x 2024-04-12T08:47:20.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:20 smithi090 ceph-mon[29100]: pgmap v176: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2024-04-12T08:47:20.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:20 smithi090 ceph-mon[29100]: mgrmap e21: y(active, since 5m), standbys: x 2024-04-12T08:47:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:20 smithi114 ceph-mon[33573]: pgmap v176: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2024-04-12T08:47:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:20 smithi114 ceph-mon[33573]: mgrmap e21: y(active, since 5m), standbys: x 2024-04-12T08:47:21.294 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:47:20] ENGINE Bus STOPPING 2024-04-12T08:47:21.294 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:47:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T08:47:21.294 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:47:20] ENGINE Bus STOPPED 2024-04-12T08:47:21.294 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:47:21] ENGINE Bus STARTING 2024-04-12T08:47:21.294 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1888984545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: mgrmap e22: x(active, starting, since 0.0144389s) 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:47:21.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: Manager daemon x is now available 2024-04-12T08:47:21.296 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:47:21.297 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:21.297 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:47:21.297 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:47:21.307 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ignoring --setuser ceph since I am not root 2024-04-12T08:47:21.307 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ignoring --setgroup ceph since I am not root 2024-04-12T08:47:21.307 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:20 smithi090 ceph-mgr[29337]: -- 172.21.15.90:0/2243990734 <== mon.2 v2:172.21.15.90:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x556e84c78c60 con 0x556e84c3ac00 2024-04-12T08:47:21.308 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:20 smithi090 ceph-mgr[29337]: -- 172.21.15.90:0/2243990734 <== mon.2 v2:172.21.15.90:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556e84c649c0 con 0x556e84c3ac00 2024-04-12T08:47:21.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1888984545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:21.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:21.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:21.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:47:21.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:47:21.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:21.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: mgrmap e22: x(active, starting, since 0.0144389s) 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:47:21.309 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: Manager daemon x is now available 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:47:21.310 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1888984545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.14421 172.21.15.90:0/4035240123' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: mgrmap e22: x(active, starting, since 0.0144389s) 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:47:21.311 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:47:21.312 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:47:21.313 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: Manager daemon x is now available 2024-04-12T08:47:21.313 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:47:21.313 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:21.313 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:47:21.313 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:47:21.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:21.289+0000 7f6f8b242700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:47:21.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:21.289+0000 7f6f8b242700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:47:21.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:21.289+0000 7f6f8b242700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:47:21.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:21.289+0000 7f6f8b242700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:47:21.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:21.289+0000 7f6f8b242700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:47:21.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:21.300+0000 7f6f9124e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:47:21.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:21.300+0000 7f6f9124e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:47:21.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:21.300+0000 7f6f9124e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:47:21.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:21.300+0000 7f6f9124e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:47:21.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:47:21.300+0000 7f6f9124e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:47:21.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:47:21] ENGINE Serving on http://:::9283 2024-04-12T08:47:21.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:47:21] ENGINE Bus STARTED 2024-04-12T08:47:22.141 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:21.672+0000 7fd33f5aa1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:47:22.141 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:21.811+0000 7fd33f5aa1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:47:22.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:22 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T08:47:22.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:22 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T08:47:22.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:22 smithi090 ceph-mon[29100]: mgrmap e23: x(active, since 1.01376s) 2024-04-12T08:47:22.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:22 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T08:47:22.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:22 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T08:47:22.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:22 smithi090 ceph-mon[40003]: mgrmap e23: x(active, since 1.01376s) 2024-04-12T08:47:22.424 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:22 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:47:22.322Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T08:47:22.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:22 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T08:47:22.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:22 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T08:47:22.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:22 smithi114 ceph-mon[33573]: mgrmap e23: x(active, since 1.01376s) 2024-04-12T08:47:23.019 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:22 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:22.749+0000 7fd33f5aa1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:47:23.019 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:22 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:22.885+0000 7fd33f5aa1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:47:23.384 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:23.018+0000 7fd33f5aa1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:47:23.384 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:23.246+0000 7fd33f5aa1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:47:23.658 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:23.383+0000 7fd33f5aa1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:47:23.659 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[29100]: Deploying cephadm binary to smithi090 2024-04-12T08:47:23.659 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[29100]: Deploying cephadm binary to smithi114 2024-04-12T08:47:23.659 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[29100]: pgmap v3: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:23.659 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:23.659 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:23.659 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3379306821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:23.659 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[40003]: Deploying cephadm binary to smithi090 2024-04-12T08:47:23.659 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[40003]: Deploying cephadm binary to smithi114 2024-04-12T08:47:23.659 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[40003]: pgmap v3: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:23.659 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:23.660 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:23.660 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3379306821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:23.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:23 smithi114 ceph-mon[33573]: Deploying cephadm binary to smithi090 2024-04-12T08:47:23.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:23 smithi114 ceph-mon[33573]: Deploying cephadm binary to smithi114 2024-04-12T08:47:23.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:23 smithi114 ceph-mon[33573]: pgmap v3: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:23.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:23 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:23.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:23 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:23.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:23 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3379306821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:23.916 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:23.657+0000 7fd33f5aa1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:47:24.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:23.915+0000 7fd33f5aa1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:47:24.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:24.032+0000 7fd33f5aa1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:47:24.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:24.176+0000 7fd33f5aa1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:47:24.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:24.312+0000 7fd33f5aa1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:47:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[40003]: pgmap v4: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[40003]: [12/Apr/2024:08:47:23] ENGINE Bus STARTING 2024-04-12T08:47:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[40003]: [12/Apr/2024:08:47:23] ENGINE Serving on http://172.21.15.114:8765 2024-04-12T08:47:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[40003]: [12/Apr/2024:08:47:23] ENGINE Serving on https://172.21.15.114:7150 2024-04-12T08:47:24.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[40003]: [12/Apr/2024:08:47:23] ENGINE Bus STARTED 2024-04-12T08:47:24.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[40003]: mgrmap e24: x(active, since 3s) 2024-04-12T08:47:24.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[29100]: pgmap v4: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:24.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[29100]: [12/Apr/2024:08:47:23] ENGINE Bus STARTING 2024-04-12T08:47:24.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[29100]: [12/Apr/2024:08:47:23] ENGINE Serving on http://172.21.15.114:8765 2024-04-12T08:47:24.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[29100]: [12/Apr/2024:08:47:23] ENGINE Serving on https://172.21.15.114:7150 2024-04-12T08:47:24.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[29100]: [12/Apr/2024:08:47:23] ENGINE Bus STARTED 2024-04-12T08:47:24.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:24 smithi090 ceph-mon[29100]: mgrmap e24: x(active, since 3s) 2024-04-12T08:47:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:24 smithi114 ceph-mon[33573]: pgmap v4: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:24 smithi114 ceph-mon[33573]: [12/Apr/2024:08:47:23] ENGINE Bus STARTING 2024-04-12T08:47:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:24 smithi114 ceph-mon[33573]: [12/Apr/2024:08:47:23] ENGINE Serving on http://172.21.15.114:8765 2024-04-12T08:47:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:24 smithi114 ceph-mon[33573]: [12/Apr/2024:08:47:23] ENGINE Serving on https://172.21.15.114:7150 2024-04-12T08:47:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:24 smithi114 ceph-mon[33573]: [12/Apr/2024:08:47:23] ENGINE Bus STARTED 2024-04-12T08:47:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:24 smithi114 ceph-mon[33573]: mgrmap e24: x(active, since 3s) 2024-04-12T08:47:25.371 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:47:25] "GET /metrics HTTP/1.1" 200 33268 "" "Prometheus/2.43.0" 2024-04-12T08:47:26.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:25 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:25.919+0000 7fd33f5aa1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:47:26.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:26 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:26.124+0000 7fd33f5aa1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:47:26.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:26 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:26.412+0000 7fd33f5aa1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:47:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:26 smithi090 ceph-mon[40003]: pgmap v5: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:26 smithi090 ceph-mon[40003]: mgrmap e25: x(active, since 4s) 2024-04-12T08:47:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:26 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2312009745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:26 smithi090 ceph-mon[29100]: pgmap v5: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:26 smithi090 ceph-mon[29100]: mgrmap e25: x(active, since 4s) 2024-04-12T08:47:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:26 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2312009745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:26 smithi114 ceph-mon[33573]: pgmap v5: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:26 smithi114 ceph-mon[33573]: mgrmap e25: x(active, since 4s) 2024-04-12T08:47:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:26 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2312009745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:26.978 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:26 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:26.720+0000 7fd33f5aa1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:47:27.261 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:26 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:26.976+0000 7fd33f5aa1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:47:27.660 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:27 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:27.260+0000 7fd33f5aa1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:47:27.922 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:27 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:27.657+0000 7fd33f5aa1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:47:27.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:27 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:27.791+0000 7fd33f5aa1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:47:28.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:28 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:28.392+0000 7fd33f5aa1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:47:28.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:28 smithi090 ceph-mon[40003]: pgmap v6: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:28.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:28 smithi090 ceph-mon[29100]: pgmap v6: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:28 smithi114 ceph-mon[33573]: pgmap v6: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-12T08:47:29.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:29 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3163999803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:29.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:29 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:29.610+0000 7fd33f5aa1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:47:29.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:29 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3163999803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:29.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:29 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3163999803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:30.417 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:30.065+0000 7fd33f5aa1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:47:30.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:30.508+0000 7fd33f5aa1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:47:30.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-mon[40003]: pgmap v7: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 561 op/s 2024-04-12T08:47:30.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-mon[29100]: pgmap v7: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 561 op/s 2024-04-12T08:47:30.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:30 smithi114 ceph-mon[33573]: pgmap v7: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 561 op/s 2024-04-12T08:47:31.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:30.805+0000 7fd33f5aa1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:47:31.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:47:30] ENGINE Bus STARTING 2024-04-12T08:47:31.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: CherryPy Checker: 2024-04-12T08:47:31.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: The Application mounted at '' has an empty config. 2024-04-12T08:47:31.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: 2024-04-12T08:47:31.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:47:30] ENGINE Serving on http://:::9283 2024-04-12T08:47:31.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: [12/Apr/2024:08:47:30] ENGINE Bus STARTED 2024-04-12T08:47:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[40003]: Standby manager daemon y started 2024-04-12T08:47:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T08:47:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:47:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T08:47:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:47:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3929849758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:31.674 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:47:31.644Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=7 2024-04-12T08:47:31.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[29100]: Standby manager daemon y started 2024-04-12T08:47:31.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T08:47:31.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:47:31.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T08:47:31.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:47:31.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3929849758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:31 smithi114 ceph-mon[33573]: Standby manager daemon y started 2024-04-12T08:47:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:31 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T08:47:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:31 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:47:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:31 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T08:47:31.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:31 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.90:0/644080039' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:47:31.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:31 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3929849758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:32.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:31 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:47:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:47:32.867 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:32 smithi090 ceph-mon[40003]: pgmap v8: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 436 op/s 2024-04-12T08:47:32.867 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:32 smithi090 ceph-mon[40003]: mgrmap e26: x(active, since 10s), standbys: y 2024-04-12T08:47:32.867 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:32 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:47:32.867 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:32 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:32.867 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:32 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:32.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:32 smithi114 ceph-mon[33573]: pgmap v8: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 436 op/s 2024-04-12T08:47:32.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:32 smithi114 ceph-mon[33573]: mgrmap e26: x(active, since 10s), standbys: y 2024-04-12T08:47:32.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:32 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:47:32.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:32 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:32.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:32 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:32.868 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:32 smithi090 ceph-mon[29100]: pgmap v8: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 436 op/s 2024-04-12T08:47:32.869 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:32 smithi090 ceph-mon[29100]: mgrmap e26: x(active, since 10s), standbys: y 2024-04-12T08:47:32.869 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:32 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:47:32.869 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:32 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:32.869 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:32 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:34 smithi114 ceph-mon[33573]: pgmap v9: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 714 op/s 2024-04-12T08:47:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:34 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3598166984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:34.883 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:34 smithi090 ceph-mon[29100]: pgmap v9: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 714 op/s 2024-04-12T08:47:34.883 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:34 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3598166984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:34.883 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:34 smithi090 ceph-mon[40003]: pgmap v9: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 714 op/s 2024-04-12T08:47:34.883 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:34 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3598166984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:35.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:47:35] "GET /metrics HTTP/1.1" 200 33268 "" "Prometheus/2.43.0" 2024-04-12T08:47:36.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:36 smithi114 ceph-mon[33573]: pgmap v10: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-12T08:47:36.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:36 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:36.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:36 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:36.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:36 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:36.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:36 smithi090 ceph-mon[29100]: pgmap v10: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-12T08:47:36.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:36 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:36.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:36 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:36.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:36 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:36 smithi090 ceph-mon[40003]: pgmap v10: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-12T08:47:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:36 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:36 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:36 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1565499333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: pgmap v11: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi090 to 2480M 2024-04-12T08:47:38.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:37 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1565499333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: pgmap v11: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-12T08:47:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi090 to 2480M 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1565499333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: pgmap v11: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:38.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:38.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi090 to 2480M 2024-04-12T08:47:38.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:37 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:39.775 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:39 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:39.775 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:39 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:39.775 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:39 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2764548860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:39.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:39 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:39.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:39 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:39.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:39 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2764548860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:39.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:39 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:39.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:39 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:39.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:39 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2764548860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:40.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:40 smithi114 ceph-mon[33573]: pgmap v12: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 995 op/s 2024-04-12T08:47:40.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:40 smithi090 ceph-mon[40003]: pgmap v12: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 995 op/s 2024-04-12T08:47:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:40 smithi090 ceph-mon[29100]: pgmap v12: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 995 op/s 2024-04-12T08:47:42.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:47:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:47:42.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:42 smithi114 ceph-mon[33573]: pgmap v13: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-12T08:47:42.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:42 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2506273042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:42 smithi090 ceph-mon[40003]: pgmap v13: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-12T08:47:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:42 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2506273042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:42.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:42 smithi090 ceph-mon[29100]: pgmap v13: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-12T08:47:42.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:42 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2506273042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:44.787 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:44 smithi114 ceph-mon[33573]: pgmap v14: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:47:44.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:44 smithi090 ceph-mon[40003]: pgmap v14: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:47:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:44 smithi090 ceph-mon[29100]: pgmap v14: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:47:45.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:47:45] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.43.0" 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/507551092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:45.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:47:45.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:45 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:47:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/507551092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:45.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:47:45.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:47:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/507551092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T08:47:45.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:45.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:47:45.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:45 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:47:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:46 smithi114 ceph-mon[33573]: Adjusting osd_memory_target on smithi114 to 2480M 2024-04-12T08:47:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:46 smithi114 ceph-mon[33573]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:47:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:46 smithi114 ceph-mon[33573]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:47:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:46 smithi114 ceph-mon[33573]: pgmap v15: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-12T08:47:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:46 smithi114 ceph-mon[33573]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:47:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:46 smithi114 ceph-mon[33573]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:47:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:46 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:46.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:46 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[29100]: Adjusting osd_memory_target on smithi114 to 2480M 2024-04-12T08:47:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[29100]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:47:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[29100]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:47:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[29100]: pgmap v15: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-12T08:47:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[29100]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:47:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[29100]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:47:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[40003]: Adjusting osd_memory_target on smithi114 to 2480M 2024-04-12T08:47:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[40003]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:47:46.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:47:46.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[40003]: pgmap v15: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-12T08:47:46.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[40003]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:47:46.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:47:46.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:46.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:46 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:47:47.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:47:47.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:47:47.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:47:47.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:47:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:47:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:47:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:47:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:47.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:48.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:48 smithi114 ceph-mon[33573]: pgmap v16: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-12T08:47:48.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:48 smithi114 ceph-mon[33573]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-12T08:47:48.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:48 smithi114 ceph-mon[33573]: Reconfiguring daemon alertmanager.a on smithi090 2024-04-12T08:47:48.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:48 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3651375636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:48.887 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:48 smithi090 ceph-mon[40003]: pgmap v16: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-12T08:47:48.887 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:48 smithi090 ceph-mon[40003]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-12T08:47:48.887 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:48 smithi090 ceph-mon[40003]: Reconfiguring daemon alertmanager.a on smithi090 2024-04-12T08:47:48.887 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:48 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3651375636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:48.887 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:48 smithi090 ceph-mon[29100]: pgmap v16: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-12T08:47:48.887 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:48 smithi090 ceph-mon[29100]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-12T08:47:48.887 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:48 smithi090 ceph-mon[29100]: Reconfiguring daemon alertmanager.a on smithi090 2024-04-12T08:47:48.887 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:48 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3651375636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:50.854 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:50 smithi090 ceph-mon[40003]: pgmap v17: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:47:50.861 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:50 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1514679780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:50.861 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:50 smithi090 ceph-mon[29100]: pgmap v17: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:47:50.861 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:50 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1514679780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:50 smithi114 ceph-mon[33573]: pgmap v17: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:47:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:50 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1514679780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:51.173 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:51 smithi090 systemd[1]: Stopping Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:47:51.837 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:51 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:51.837 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:51 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:51.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:51 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:47:52.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:47:51 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:47:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:47:52.173 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:51 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[73302]: ts=2024-04-12T08:47:51.846Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-12T08:47:52.173 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:51 smithi090 podman[85139]: 2024-04-12 08:47:51.899276156 +0000 UTC m=+0.635695479 container died 0893314a69712e380d603b3415ef7bf447ee0f3e9373b8b45246d4126cff7312 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:52.545 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:52 smithi090 ceph-mon[29100]: pgmap v18: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-12T08:47:52.545 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:52 smithi090 podman[85139]: 2024-04-12 08:47:52.397401734 +0000 UTC m=+1.133821047 container cleanup 0893314a69712e380d603b3415ef7bf447ee0f3e9373b8b45246d4126cff7312 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:52.545 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:52 smithi090 bash[85139]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a 2024-04-12T08:47:52.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:52 smithi114 ceph-mon[33573]: pgmap v18: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-12T08:47:52.617 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:47:52 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:47:52.315Z caller=notifier.go:532 level=error component=notifier alertmanager=http://smithi090.front.sepia.ceph.com:9093/api/v2/alerts count=1 msg="Error sending alert" err="Post \"http://smithi090.front.sepia.ceph.com:9093/api/v2/alerts\": dial tcp 172.21.15.90:9093: connect: connection refused" 2024-04-12T08:47:52.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:52 smithi090 ceph-mon[40003]: pgmap v18: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-12T08:47:52.923 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:52 smithi090 podman[85149]: 2024-04-12 08:47:52.684375051 +0000 UTC m=+0.784389729 container remove 0893314a69712e380d603b3415ef7bf447ee0f3e9373b8b45246d4126cff7312 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:52.923 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:52 smithi090 podman[85149]: 2024-04-12 08:47:52.779163965 +0000 UTC m=+0.879178652 volume remove 09e6f45e6bbba25d243daf60d1a660b8346bbe53a4aaf0bb2926f86cb9c5f48d 2024-04-12T08:47:53.318 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-12T08:47:53.318 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 systemd[1]: Stopped Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:47:53.319 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@alertmanager.a.service: Consumed 1.309s CPU time. 2024-04-12T08:47:53.319 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 systemd[1]: Starting Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:47:53.658 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-mon[29100]: pgmap v19: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T08:47:53.658 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3990110953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:53.658 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-mon[40003]: pgmap v19: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T08:47:53.659 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3990110953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:53.659 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 podman[85250]: 2024-04-12 08:47:53.222577432 +0000 UTC m=+0.031717995 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-12T08:47:53.659 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 podman[85250]: 2024-04-12 08:47:53.328768154 +0000 UTC m=+0.137908704 volume create 9828ec39c157dd62a36a107b677660ce700a1d3837386005240dfd01343651b8 2024-04-12T08:47:53.659 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 podman[85250]: 2024-04-12 08:47:53.446336269 +0000 UTC m=+0.255476826 container create d11ba2dc752ae610824e54b7c3d94723bb8d055d163782c66b9f92559ea157f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:53 smithi114 ceph-mon[33573]: pgmap v19: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T08:47:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:53 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3990110953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:53.923 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 podman[85250]: 2024-04-12 08:47:53.785331117 +0000 UTC m=+0.594471676 container init d11ba2dc752ae610824e54b7c3d94723bb8d055d163782c66b9f92559ea157f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:53.923 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 podman[85250]: 2024-04-12 08:47:53.788891515 +0000 UTC m=+0.598032074 container start d11ba2dc752ae610824e54b7c3d94723bb8d055d163782c66b9f92559ea157f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:53.924 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:47:53.806Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-12T08:47:53.924 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:47:53.807Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-12T08:47:53.924 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:47:53.811Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.90 port=9094 2024-04-12T08:47:53.924 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:47:53.814Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-12T08:47:53.924 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:47:53.850Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-12T08:47:53.924 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:47:53.851Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-12T08:47:53.924 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:47:53.854Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-12T08:47:53.924 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:47:53.854Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-12T08:47:53.924 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 bash[85250]: d11ba2dc752ae610824e54b7c3d94723bb8d055d163782c66b9f92559ea157f4 2024-04-12T08:47:53.925 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:53 smithi090 systemd[1]: Started Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:47:55.136 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:54 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:55.137 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:54 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:55.137 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:54 smithi090 ceph-mon[29100]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-12T08:47:55.137 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:54 smithi090 ceph-mon[29100]: Reconfiguring daemon node-exporter.a on smithi090 2024-04-12T08:47:55.137 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:54 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:55.137 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:54 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:55.137 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:54 smithi090 ceph-mon[40003]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-12T08:47:55.137 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:54 smithi090 ceph-mon[40003]: Reconfiguring daemon node-exporter.a on smithi090 2024-04-12T08:47:55.137 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:54 smithi090 systemd[1]: Stopping Ceph node-exporter.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:47:55.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:54 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:55.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:54 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:55.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:54 smithi114 ceph-mon[33573]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-12T08:47:55.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:54 smithi114 ceph-mon[33573]: Reconfiguring daemon node-exporter.a on smithi090 2024-04-12T08:47:55.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:47:55 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:47:55] "GET /metrics HTTP/1.1" 200 36198 "" "Prometheus/2.43.0" 2024-04-12T08:47:55.423 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:55 smithi090 podman[85391]: 2024-04-12 08:47:55.136039184 +0000 UTC m=+0.165757592 container died b71f7d0a7569f877254bcb7090537fe2aeaabbbc491e6c5b24dfd615b7b7629e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:55.815 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:55 smithi090 podman[85391]: 2024-04-12 08:47:55.710578845 +0000 UTC m=+0.740297260 container cleanup b71f7d0a7569f877254bcb7090537fe2aeaabbbc491e6c5b24dfd615b7b7629e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:55.815 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:55 smithi090 bash[85391]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a 2024-04-12T08:47:56.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:55 smithi090 ceph-mon[40003]: pgmap v20: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:47:56.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:55 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2651376500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:56.173 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:47:55 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:47:55.815Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000589556s 2024-04-12T08:47:56.173 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 podman[85405]: 2024-04-12 08:47:56.02130923 +0000 UTC m=+0.887619989 container remove b71f7d0a7569f877254bcb7090537fe2aeaabbbc491e6c5b24dfd615b7b7629e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:56.173 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-12T08:47:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:55 smithi090 ceph-mon[29100]: pgmap v20: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:47:56.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:55 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2651376500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:55 smithi114 ceph-mon[33573]: pgmap v20: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:47:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:55 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2651376500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:56.516 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-12T08:47:56.516 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 systemd[1]: Stopped Ceph node-exporter.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:47:56.516 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.a.service: Consumed 2.496s CPU time. 2024-04-12T08:47:56.516 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 systemd[1]: Starting Ceph node-exporter.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:47:56.803 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 podman[85507]: 2024-04-12 08:47:56.41983789 +0000 UTC m=+0.038359248 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-12T08:47:56.803 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 podman[85507]: 2024-04-12 08:47:56.55718885 +0000 UTC m=+0.175710271 container create 687747c6880c9b137242934886c0c817b6c8e462738e3f58f0fbf0f2bf7881d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:56.803 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 podman[85507]: 2024-04-12 08:47:56.803322796 +0000 UTC m=+0.421844154 container init 687747c6880c9b137242934886c0c817b6c8e462738e3f58f0fbf0f2bf7881d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:57.173 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 podman[85507]: 2024-04-12 08:47:56.808362785 +0000 UTC m=+0.426884138 container start 687747c6880c9b137242934886c0c817b6c8e462738e3f58f0fbf0f2bf7881d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T08:47:57.173 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.813Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-12T08:47:57.173 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.813Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.815Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.816Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.816Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.816Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.817Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.817Z caller=node_exporter.go:117 level=info collector=arp 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.817Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.817Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.817Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.817Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.817Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.817Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-12T08:47:57.174 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-12T08:47:57.175 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=edac 2024-04-12T08:47:57.175 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-12T08:47:57.175 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-12T08:47:57.175 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-12T08:47:57.175 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-12T08:47:57.175 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-12T08:47:57.175 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=os 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-12T08:47:57.176 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=stat 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=time 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=uname 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.818Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.819Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a[85519]: ts=2024-04-12T08:47:56.819Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-12T08:47:57.177 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 bash[85507]: 687747c6880c9b137242934886c0c817b6c8e462738e3f58f0fbf0f2bf7881d6 2024-04-12T08:47:57.178 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 08:47:56 smithi090 systemd[1]: Started Ceph node-exporter.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:47:58.331 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:57 smithi114 ceph-mon[33573]: pgmap v21: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:47:58.331 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:57 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:58.331 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:57 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:58.331 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:57 smithi114 ceph-mon[33573]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T08:47:58.331 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:57 smithi114 ceph-mon[33573]: Reconfiguring daemon prometheus.a on smithi114 2024-04-12T08:47:58.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:57 smithi090 ceph-mon[29100]: pgmap v21: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:47:58.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:57 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:58.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:57 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:58.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:57 smithi090 ceph-mon[29100]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T08:47:58.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:57 smithi090 ceph-mon[29100]: Reconfiguring daemon prometheus.a on smithi114 2024-04-12T08:47:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:57 smithi090 ceph-mon[40003]: pgmap v21: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:47:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:57 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:57 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:47:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:57 smithi090 ceph-mon[40003]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T08:47:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:57 smithi090 ceph-mon[40003]: Reconfiguring daemon prometheus.a on smithi114 2024-04-12T08:47:59.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:58 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2792367608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:59.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:58 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2792367608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:47:59.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:58 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2792367608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:47:59 smithi114 ceph-mon[33573]: pgmap v22: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-12T08:48:00.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:47:59 smithi090 ceph-mon[40003]: pgmap v22: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-12T08:48:00.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:47:59 smithi090 ceph-mon[29100]: pgmap v22: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-12T08:48:01.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:01 smithi114 systemd[1]: Stopping Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:48:02.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:01 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[29333]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:48:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:48:02.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:01 smithi090 ceph-mon[40003]: pgmap v23: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-12T08:48:02.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:01 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1718674090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:02.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:01 smithi090 ceph-mon[29100]: pgmap v23: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-12T08:48:02.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:01 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1718674090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:02.343 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.027Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-12T08:48:02.343 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.028Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-12T08:48:02.343 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.028Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-12T08:48:02.343 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.028Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-12T08:48:02.343 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.028Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-12T08:48:02.343 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.028Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-12T08:48:02.343 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.028Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-12T08:48:02.343 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.028Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-12T08:48:02.343 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.030Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-12T08:48:02.344 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.030Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-12T08:48:02.344 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.030Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-12T08:48:02.344 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[63723]: ts=2024-04-12T08:48:02.030Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-12T08:48:02.344 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 podman[75553]: 2024-04-12 08:48:02.036434503 +0000 UTC m=+0.174267803 container died 94ac082be83bd7d49452d206a5b86ed9ab3584efb8b6c8acf6270c0147e544d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:48:02.344 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:01 smithi114 ceph-mon[33573]: pgmap v23: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-12T08:48:02.344 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:01 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1718674090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:02.617 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 podman[75553]: 2024-04-12 08:48:02.534395678 +0000 UTC m=+0.672228962 container cleanup 94ac082be83bd7d49452d206a5b86ed9ab3584efb8b6c8acf6270c0147e544d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:48:02.617 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 bash[75553]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a 2024-04-12T08:48:03.367 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:02 smithi114 podman[75564]: 2024-04-12 08:48:02.969912663 +0000 UTC m=+0.928631485 container remove 94ac082be83bd7d49452d206a5b86ed9ab3584efb8b6c8acf6270c0147e544d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:48:03.367 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-12T08:48:03.367 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 systemd[1]: Stopped Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:03.367 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@prometheus.a.service: Consumed 2.179s CPU time. 2024-04-12T08:48:03.367 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 systemd[1]: Starting Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:48:03.818 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 podman[75641]: 2024-04-12 08:48:03.436398892 +0000 UTC m=+0.041914497 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-12T08:48:03.818 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 podman[75641]: 2024-04-12 08:48:03.554538471 +0000 UTC m=+0.160054069 container create 0765e8b7f635952396ab1620fcb9a055370307ee34a157b90d988de3cd906f73 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:48:04.075 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:48:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:48:03.818Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003295103s 2024-04-12T08:48:04.075 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:03 smithi090 ceph-mon[40003]: pgmap v24: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-12T08:48:04.075 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:03 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1416435296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:04.075 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:03 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:04.075 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:03 smithi090 ceph-mon[29100]: pgmap v24: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-12T08:48:04.075 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:03 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1416435296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:04.075 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:03 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-mon[33573]: pgmap v24: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-12T08:48:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1416435296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:04.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 podman[75641]: 2024-04-12 08:48:03.81799119 +0000 UTC m=+0.423506800 container init 0765e8b7f635952396ab1620fcb9a055370307ee34a157b90d988de3cd906f73 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:48:04.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 podman[75641]: 2024-04-12 08:48:03.825273219 +0000 UTC m=+0.430788818 container start 0765e8b7f635952396ab1620fcb9a055370307ee34a157b90d988de3cd906f73 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.870Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.870Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.870Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi114 (none))" 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.870Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.870Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.872Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.873Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.877Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.877Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.885Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.885Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.288µs 2024-04-12T08:48:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.885Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-12T08:48:04.119 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.885Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-12T08:48:04.119 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 bash[75641]: 0765e8b7f635952396ab1620fcb9a055370307ee34a157b90d988de3cd906f73 2024-04-12T08:48:04.119 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.919Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-12T08:48:04.119 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.919Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-12T08:48:04.119 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.919Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=36.346µs wal_replay_duration=33.97842ms wbl_replay_duration=158ns total_replay_duration=34.258649ms 2024-04-12T08:48:04.119 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 systemd[1]: Started Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:04.119 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.923Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-12T08:48:04.119 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.923Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-12T08:48:04.119 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.923Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-12T08:48:04.119 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.961Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=38.271242ms db_storage=1.153µs remote_storage=1.673µs web_handler=522ns query_engine=1.036µs scrape=416.729µs scrape_sd=148.164µs notify=15.32µs notify_sd=14.787µs rules=37.306541ms tracing=8.64µs 2024-04-12T08:48:04.120 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.962Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-12T08:48:04.120 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:03.962Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-12T08:48:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:05 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:05 smithi114 ceph-mon[33573]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-12T08:48:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:05 smithi114 ceph-mon[33573]: Reconfiguring daemon node-exporter.b on smithi114 2024-04-12T08:48:05.117 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:04 smithi114 systemd[1]: Stopping Ceph node-exporter.b for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:48:05.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:05 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:05.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:05 smithi090 ceph-mon[40003]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-12T08:48:05.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:05 smithi090 ceph-mon[40003]: Reconfiguring daemon node-exporter.b on smithi114 2024-04-12T08:48:05.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:05 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:05.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:05 smithi090 ceph-mon[29100]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-12T08:48:05.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:05 smithi090 ceph-mon[29100]: Reconfiguring daemon node-exporter.b on smithi114 2024-04-12T08:48:05.617 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:05 smithi114 podman[75780]: 2024-04-12 08:48:05.324267218 +0000 UTC m=+0.212972206 container died 9a6215e32fcfa99b2b91e57343f9dcbd81202a8e95dd84211aa6a9e658ac457c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T08:48:05.999 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:05 smithi114 podman[75780]: 2024-04-12 08:48:05.73928742 +0000 UTC m=+0.627992414 container cleanup 9a6215e32fcfa99b2b91e57343f9dcbd81202a8e95dd84211aa6a9e658ac457c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T08:48:05.999 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:05 smithi114 bash[75780]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b 2024-04-12T08:48:06.367 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:05 smithi114 podman[75791]: 2024-04-12 08:48:05.999331144 +0000 UTC m=+0.672305609 container remove 9a6215e32fcfa99b2b91e57343f9dcbd81202a8e95dd84211aa6a9e658ac457c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T08:48:06.367 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:06 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-12T08:48:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:06 smithi114 ceph-mon[33573]: pgmap v25: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-12T08:48:06.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:06 smithi090 ceph-mon[40003]: pgmap v25: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-12T08:48:06.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:06 smithi090 ceph-mon[29100]: pgmap v25: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-12T08:48:06.772 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:06 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-12T08:48:06.772 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:06 smithi114 systemd[1]: Stopped Ceph node-exporter.b for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:06.772 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:06 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.b.service: Consumed 2.508s CPU time. 2024-04-12T08:48:06.772 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:06 smithi114 systemd[1]: Starting Ceph node-exporter.b for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:48:07.078 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:07.079 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1283702718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:07.079 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:06 smithi114 podman[75869]: 2024-04-12 08:48:06.676124641 +0000 UTC m=+0.040634755 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-12T08:48:07.079 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:06 smithi114 podman[75869]: 2024-04-12 08:48:06.837460623 +0000 UTC m=+0.201970723 container create e89762981b5565d3e57c7956c584040f732e7bad59524c54d27526b2284c879c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T08:48:07.079 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 podman[75869]: 2024-04-12 08:48:07.078715155 +0000 UTC m=+0.443225262 container init e89762981b5565d3e57c7956c584040f732e7bad59524c54d27526b2284c879c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T08:48:07.367 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 podman[75869]: 2024-04-12 08:48:07.089134378 +0000 UTC m=+0.453644486 container start e89762981b5565d3e57c7956c584040f732e7bad59524c54d27526b2284c879c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.096Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.097Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.098Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.098Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.098Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.098Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=arp 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-12T08:48:07.368 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-12T08:48:07.369 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-12T08:48:07.369 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-12T08:48:07.369 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-12T08:48:07.369 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-12T08:48:07.369 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=edac 2024-04-12T08:48:07.369 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-12T08:48:07.369 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-12T08:48:07.369 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-12T08:48:07.370 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=os 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=stat 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=time 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-12T08:48:07.371 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=uname 2024-04-12T08:48:07.372 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-12T08:48:07.372 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-12T08:48:07.372 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-12T08:48:07.372 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-12T08:48:07.372 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b[75881]: ts=2024-04-12T08:48:07.099Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-12T08:48:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:07 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:07.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:07 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1283702718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:07.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:07 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:07.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:07 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1283702718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:07.846 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 bash[75869]: e89762981b5565d3e57c7956c584040f732e7bad59524c54d27526b2284c879c 2024-04-12T08:48:07.846 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 08:48:07 smithi114 systemd[1]: Started Ceph node-exporter.b for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:08 smithi114 ceph-mon[33573]: pgmap v26: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-12T08:48:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:08 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:08 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:08.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:08 smithi090 ceph-mon[40003]: pgmap v26: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-12T08:48:08.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:08 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:08.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:08 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:08.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:08 smithi090 ceph-mon[29100]: pgmap v26: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-12T08:48:08.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:08 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:08.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:08 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:09.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:09 smithi114 ceph-mon[33573]: Reconfiguring grafana.a (dependencies changed)... 2024-04-12T08:48:09.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:09 smithi114 ceph-mon[33573]: Reconfiguring daemon grafana.a on smithi114 2024-04-12T08:48:09.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:09 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1913909178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:09.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:09 smithi090 ceph-mon[40003]: Reconfiguring grafana.a (dependencies changed)... 2024-04-12T08:48:09.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:09 smithi090 ceph-mon[40003]: Reconfiguring daemon grafana.a on smithi114 2024-04-12T08:48:09.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:09 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1913909178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:09.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:09 smithi090 ceph-mon[29100]: Reconfiguring grafana.a (dependencies changed)... 2024-04-12T08:48:09.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:09 smithi090 ceph-mon[29100]: Reconfiguring daemon grafana.a on smithi114 2024-04-12T08:48:09.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:09 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1913909178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:10.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:10 smithi114 ceph-mon[33573]: pgmap v27: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:48:10.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:10 smithi090 ceph-mon[29100]: pgmap v27: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:48:10.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:10 smithi090 ceph-mon[40003]: pgmap v27: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:48:12.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:12 smithi114 ceph-mon[33573]: pgmap v28: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-04-12T08:48:12.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:12 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1336180906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:12.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:12 smithi090 ceph-mon[40003]: pgmap v28: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-04-12T08:48:12.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:12 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1336180906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:12.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:12 smithi090 ceph-mon[29100]: pgmap v28: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-04-12T08:48:12.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:12 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1336180906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:13.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:13 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:48:13] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.43.0" 2024-04-12T08:48:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:14 smithi114 ceph-mon[33573]: pgmap v29: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:48:14.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:14 smithi090 ceph-mon[40003]: pgmap v29: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:48:14.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:14 smithi090 ceph-mon[29100]: pgmap v29: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:48:15.295 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:15 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2586805714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:15.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:15 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2586805714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:15.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:15 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2586805714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:16 smithi114 ceph-mon[33573]: pgmap v30: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-12T08:48:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:16 smithi090 ceph-mon[40003]: pgmap v30: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-12T08:48:16.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:16 smithi090 ceph-mon[29100]: pgmap v30: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-12T08:48:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:17 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/191419297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:17.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:17 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/191419297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:17.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:17 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/191419297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:18.324 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:18 smithi090 ceph-mon[29100]: pgmap v31: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-04-12T08:48:18.324 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:18 smithi090 ceph-mon[40003]: pgmap v31: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-04-12T08:48:18.363 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:18 smithi114 ceph-mon[33573]: pgmap v31: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-04-12T08:48:19.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:48:20.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:20 smithi114 ceph-mon[33573]: pgmap v32: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 974 op/s 2024-04-12T08:48:20.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:20 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1312944506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:20.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:20 smithi090 ceph-mon[29100]: pgmap v32: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 974 op/s 2024-04-12T08:48:20.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:20 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1312944506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:20.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:20 smithi090 ceph-mon[40003]: pgmap v32: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 974 op/s 2024-04-12T08:48:20.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:20 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1312944506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:21.117 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:20 smithi114 systemd[1]: Stopping Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:48:21.755 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:21 smithi114 podman[76163]: 2024-04-12 08:48:21.458413072 +0000 UTC m=+0.268063579 container died 44058eaefb748e9b75b348f9d15698ae73638d871d03120925904a0e4d27ec6c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, name=ubi8, release=239.1651231664, architecture=x86_64, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.buildah.version=1.32.0, version=8.5, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vendor=Red Hat, Inc.) 2024-04-12T08:48:22.061 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:21 smithi114 podman[76163]: 2024-04-12 08:48:21.951969135 +0000 UTC m=+0.761619642 container cleanup 44058eaefb748e9b75b348f9d15698ae73638d871d03120925904a0e4d27ec6c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.buildah.version=1.32.0, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , version=8.5, io.openshift.expose-services=, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, name=ubi8, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container) 2024-04-12T08:48:22.061 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:21 smithi114 bash[76163]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a 2024-04-12T08:48:22.313 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:22 smithi114 ceph-mon[33573]: pgmap v33: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2024-04-12T08:48:22.313 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:22 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:22.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:22 smithi090 ceph-mon[40003]: pgmap v33: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2024-04-12T08:48:22.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:22 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:22.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:22 smithi090 ceph-mon[29100]: pgmap v33: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2024-04-12T08:48:22.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:22 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:22.570 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:22 smithi114 podman[76175]: 2024-04-12 08:48:22.31322745 +0000 UTC m=+0.854547335 container remove 44058eaefb748e9b75b348f9d15698ae73638d871d03120925904a0e4d27ec6c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, description=Ceph Grafana Container, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., io.openshift.expose-services=, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git) 2024-04-12T08:48:22.570 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:22 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-12T08:48:22.867 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:22 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:48:22.867 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:22 smithi114 systemd[1]: Stopped Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:22.867 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:22 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Consumed 6.822s CPU time. 2024-04-12T08:48:22.868 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:22 smithi114 systemd[1]: Starting Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:48:23.156 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:23 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1224317517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:23.157 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:22 smithi114 podman[76253]: 2024-04-12 08:48:22.794919174 +0000 UTC m=+0.029892494 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T08:48:23.157 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:22 smithi114 podman[76253]: 2024-04-12 08:48:22.932179755 +0000 UTC m=+0.167153042 container create 87a0031046707448489f5a06b716c4a258aa61e214272c2fc91918975ced5898 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Paul Cuzner , version=8.5, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, vendor=Red Hat, Inc.) 2024-04-12T08:48:23.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:23 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1224317517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:23.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:23 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1224317517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:23.435 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:23 smithi114 podman[76253]: 2024-04-12 08:48:23.335006981 +0000 UTC m=+0.569980274 container init 87a0031046707448489f5a06b716c4a258aa61e214272c2fc91918975ced5898 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , vcs-type=git, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base rhel8, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.32.0, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8) 2024-04-12T08:48:23.435 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:23 smithi114 podman[76253]: 2024-04-12 08:48:23.339447994 +0000 UTC m=+0.574421282 container start 87a0031046707448489f5a06b716c4a258aa61e214272c2fc91918975ced5898 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, architecture=x86_64, vcs-type=git, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, version=8.5, name=ubi8) 2024-04-12T08:48:23.435 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:23 smithi114 bash[76253]: 87a0031046707448489f5a06b716c4a258aa61e214272c2fc91918975ced5898 2024-04-12T08:48:23.721 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:48:23] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.43.0" 2024-04-12T08:48:23.721 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:23 smithi114 systemd[1]: Started Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:23.721 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[76265]: logger=settings t=2024-04-12T08:48:23.659633732Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-12T08:48:23.721 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:23 smithi114 podman[76282]: 2024-04-12 08:48:23.720622009 +0000 UTC m=+0.037280582 container died 87a0031046707448489f5a06b716c4a258aa61e214272c2fc91918975ced5898 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.openshift.expose-services=, architecture=x86_64, vcs-type=git, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-12T08:48:24.801 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:24 smithi114 ceph-mon[33573]: pgmap v34: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-04-12T08:48:24.801 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:24 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:24.801 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:24 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:24.801 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:24 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:48:24.801 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:24 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:48:24.801 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:24 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:48:24.801 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:24 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:24.802 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:24 smithi114 podman[76282]: 2024-04-12 08:48:24.407780447 +0000 UTC m=+0.724439009 container remove 87a0031046707448489f5a06b716c4a258aa61e214272c2fc91918975ced5898 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , distribution-scope=public, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, com.redhat.component=ubi8-container, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., version=8.5, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vcs-type=git) 2024-04-12T08:48:24.802 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:24 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-12T08:48:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[40003]: pgmap v34: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-04-12T08:48:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:48:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:48:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:48:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:24.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[29100]: pgmap v34: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-04-12T08:48:24.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:24.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:24.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:48:24.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:48:24.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:48:24.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:24 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:25.117 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:24 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:48:25.117 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:24 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Consumed 1.048s CPU time. 2024-04-12T08:48:25.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[40003]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:48:25.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[40003]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:48:25.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[40003]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:48:25.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[40003]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T08:48:25.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[40003]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T08:48:25.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1004898188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:25.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[29100]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:48:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[29100]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:48:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[29100]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:48:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[29100]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T08:48:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[29100]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T08:48:25.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:25 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1004898188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:25 smithi114 ceph-mon[33573]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T08:48:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:25 smithi114 ceph-mon[33573]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:48:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:25 smithi114 ceph-mon[33573]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T08:48:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:25 smithi114 ceph-mon[33573]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T08:48:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:25 smithi114 ceph-mon[33573]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T08:48:25.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:25 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1004898188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:26 smithi114 ceph-mon[33573]: pgmap v35: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-12T08:48:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:26 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:26 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:48:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:26 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:48:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:26 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:48:26.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:26 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:48:26.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[29100]: pgmap v35: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-12T08:48:26.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:26.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:48:26.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:48:26.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:48:26.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:48:26.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[40003]: pgmap v35: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-12T08:48:26.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:26.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:48:26.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:48:26.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:48:26.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:26 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:48:27.767 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:27 smithi090 ceph-mon[40003]: Upgrade: Updating mgr.y 2024-04-12T08:48:27.767 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:27 smithi090 ceph-mon[40003]: Deploying daemon mgr.y on smithi090 2024-04-12T08:48:27.767 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:27 smithi090 ceph-mon[29100]: Upgrade: Updating mgr.y 2024-04-12T08:48:27.767 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:27 smithi090 ceph-mon[29100]: Deploying daemon mgr.y on smithi090 2024-04-12T08:48:27.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:27 smithi114 ceph-mon[33573]: Upgrade: Updating mgr.y 2024-04-12T08:48:27.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:27 smithi114 ceph-mon[33573]: Deploying daemon mgr.y on smithi090 2024-04-12T08:48:28.829 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:28 smithi090 ceph-mon[29100]: pgmap v36: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-12T08:48:28.829 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:28 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3348792193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:28.829 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:28 smithi090 ceph-mon[40003]: pgmap v36: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-12T08:48:28.829 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:28 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3348792193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:28.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:48:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:28 smithi114 ceph-mon[33573]: pgmap v36: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-12T08:48:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:28 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3348792193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:29.640 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:29 smithi090 systemd[1]: Stopping Ceph mgr.y for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:48:29.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:29 smithi090 podman[86110]: 2024-04-12 08:48:29.691740629 +0000 UTC m=+0.194946805 container died 7ed4ce011c21e174b773bcddfd4cdbd37fbe293762c1ee70020c6f1c6b3d9a18 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-12T08:48:30.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:30 smithi114 ceph-mon[33573]: pgmap v37: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:48:30.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:30 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1527794435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:30.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:30 smithi090 ceph-mon[40003]: pgmap v37: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:48:30.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:30 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1527794435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:30.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:30 smithi090 ceph-mon[29100]: pgmap v37: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:48:30.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:30 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1527794435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:32.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:31 smithi090 podman[86110]: 2024-04-12 08:48:31.863886195 +0000 UTC m=+2.367092381 container cleanup 7ed4ce011c21e174b773bcddfd4cdbd37fbe293762c1ee70020c6f1c6b3d9a18 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-12T08:48:32.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:31 smithi090 bash[86110]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y 2024-04-12T08:48:32.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:32 smithi090 podman[86122]: 2024-04-12 08:48:32.104075684 +0000 UTC m=+2.406905416 container remove 7ed4ce011c21e174b773bcddfd4cdbd37fbe293762c1ee70020c6f1c6b3d9a18 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-12T08:48:32.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:32 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-12T08:48:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:32 smithi114 ceph-mon[33573]: pgmap v38: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-12T08:48:32.897 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:32 smithi090 ceph-mon[40003]: pgmap v38: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-12T08:48:32.898 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:32 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-12T08:48:32.898 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:32 smithi090 systemd[1]: Stopped Ceph mgr.y for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:32.898 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:32 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.y.service: Consumed 1min 2.846s CPU time. 2024-04-12T08:48:32.898 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:32 smithi090 systemd[1]: Starting Ceph mgr.y for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:48:32.898 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:32 smithi090 ceph-mon[29100]: pgmap v38: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-12T08:48:33.508 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:33 smithi090 podman[86284]: 2024-04-12 08:48:33.14619791 +0000 UTC m=+0.030475406 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:48:33.508 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:33 smithi090 podman[86284]: 2024-04-12 08:48:33.277936704 +0000 UTC m=+0.162214236 container create b61a0f582c618169641e6300aedbf97a39978538f65549d8f235bf0b021da1de (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.vendor=CentOS) 2024-04-12T08:48:33.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:33 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3528567547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:33.868 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:33 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:48:33] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.43.0" 2024-04-12T08:48:33.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:33 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3528567547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:33.923 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:48:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:48:33.507Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T08:48:33.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:33 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3528567547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:33.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:33 smithi090 podman[86284]: 2024-04-12 08:48:33.540432564 +0000 UTC m=+0.424710068 container init b61a0f582c618169641e6300aedbf97a39978538f65549d8f235bf0b021da1de (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-12T08:48:33.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:33 smithi090 podman[86284]: 2024-04-12 08:48:33.544253265 +0000 UTC m=+0.428530762 container start b61a0f582c618169641e6300aedbf97a39978538f65549d8f235bf0b021da1de (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:48:33.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:33 smithi090 ceph-mgr[86301]: -- 172.21.15.90:0/1415369887 <== mon.2 v2:172.21.15.90:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557e7f6965a0 con 0x557e7f66b000 2024-04-12T08:48:33.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:33 smithi090 bash[86284]: b61a0f582c618169641e6300aedbf97a39978538f65549d8f235bf0b021da1de 2024-04-12T08:48:33.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:33 smithi090 systemd[1]: Started Ceph mgr.y for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:34 smithi114 ceph-mon[33573]: pgmap v39: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:48:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:34 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:34 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:34 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:34.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:34 smithi090 ceph-mon[40003]: pgmap v39: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:48:34.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:34 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:34.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:34 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:34.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:34 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:34.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:34 smithi090 ceph-mon[29100]: pgmap v39: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-12T08:48:34.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:34 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:34.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:34 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:34.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:34 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:34.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:34 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:34.466+0000 7f35cd61f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:48:34.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:34 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:34.631+0000 7f35cd61f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:48:35.127 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:34 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-12T08:48:35.127 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:34 smithi114 systemd[1]: Stopped Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:35.127 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:34 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Consumed 1.048s CPU time. 2024-04-12T08:48:35.127 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:34 smithi114 systemd[1]: Starting Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:48:35.584 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:35 smithi114 podman[76513]: 2024-04-12 08:48:35.125594105 +0000 UTC m=+0.040670006 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T08:48:35.584 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:35 smithi114 podman[76513]: 2024-04-12 08:48:35.244790026 +0000 UTC m=+0.159866003 container create d512b1559a021f080da3d55240d0db727a1f5553194fc6e7e401448878f72edf (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.buildah.version=1.32.0, release=239.1651231664, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.openshift.expose-services=, maintainer=Paul Cuzner , name=ubi8, vcs-type=git, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container) 2024-04-12T08:48:35.848 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:35 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:35.848 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:35 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:35.848 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:35 smithi114 ceph-mon[33573]: pgmap v40: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-04-12T08:48:35.848 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:35 smithi114 podman[76513]: 2024-04-12 08:48:35.740232659 +0000 UTC m=+0.655308555 container init d512b1559a021f080da3d55240d0db727a1f5553194fc6e7e401448878f72edf (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, architecture=x86_64, vendor=Red Hat, Inc., description=Ceph Grafana Container, vcs-type=git, name=ubi8, version=8.5, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , distribution-scope=public, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-12T08:48:35.848 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:35 smithi114 podman[76513]: 2024-04-12 08:48:35.744811906 +0000 UTC m=+0.659887802 container start d512b1559a021f080da3d55240d0db727a1f5553194fc6e7e401448878f72edf (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.openshift.tags=base rhel8, release=239.1651231664, vcs-type=git, io.openshift.expose-services=, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, version=8.5, maintainer=Paul Cuzner , name=ubi8, vendor=Red Hat, Inc., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-12T08:48:35.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:35 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:35.794+0000 7f35cd61f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:48:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:35 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:35 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:35.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:35 smithi090 ceph-mon[29100]: pgmap v40: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-04-12T08:48:35.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:35 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:35.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:35 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:35.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:35 smithi090 ceph-mon[40003]: pgmap v40: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-04-12T08:48:36.121 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:35 smithi114 bash[76513]: d512b1559a021f080da3d55240d0db727a1f5553194fc6e7e401448878f72edf 2024-04-12T08:48:36.121 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:35 smithi114 systemd[1]: Started Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:36.121 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[76538]: logger=settings t=2024-04-12T08:48:35.999195322Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-12T08:48:36.121 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:36 smithi114 podman[76580]: 2024-04-12 08:48:36.044385189 +0000 UTC m=+0.032227330 container died d512b1559a021f080da3d55240d0db727a1f5553194fc6e7e401448878f72edf (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.buildah.version=1.32.0, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, vcs-type=git, description=Ceph Grafana Container, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., name=ubi8, release=239.1651231664) 2024-04-12T08:48:36.248 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:35 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:35.956+0000 7f35cd61f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:48:36.248 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:36.181+0000 7f35cd61f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:48:36.524 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:36.379+0000 7f35cd61f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:48:36.866 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:36.523+0000 7f35cd61f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:48:36.866 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:36 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4104089143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:36.866 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:36 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:36.873 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:36 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4104089143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:36.873 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:36 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:36 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4104089143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:36 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:37.118 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:36 smithi114 podman[76580]: 2024-04-12 08:48:36.724720636 +0000 UTC m=+0.712562767 container remove d512b1559a021f080da3d55240d0db727a1f5553194fc6e7e401448878f72edf (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., architecture=x86_64, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, release=239.1651231664) 2024-04-12T08:48:37.118 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:36 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-12T08:48:37.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:36.982+0000 7f35cd61f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:48:37.482 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:37 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:37.176+0000 7f35cd61f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:48:37.482 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:37 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:37.331+0000 7f35cd61f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:48:37.482 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:37 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:37.480+0000 7f35cd61f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:48:37.746 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:37 smithi090 ceph-mon[40003]: pgmap v41: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-04-12T08:48:37.746 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:37 smithi090 ceph-mon[29100]: pgmap v41: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-04-12T08:48:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:37 smithi114 ceph-mon[33573]: pgmap v41: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-04-12T08:48:38.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:37 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:37.747+0000 7f35cd61f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:48:38.722 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:38 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2647900220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:38.723 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:38 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2647900220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:38.867 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:38 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:48:38.867 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:38 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Consumed 1.034s CPU time. 2024-04-12T08:48:38.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:38 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2647900220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:38.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:48:39.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:39 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:39.370+0000 7f35cd61f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:48:39.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:39 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:39.598+0000 7f35cd61f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:48:40.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:40 smithi114 ceph-mon[33573]: pgmap v42: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:48:40.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:40 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:40.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:40 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:40.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:40.144+0000 7f35cd61f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:48:40.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:40.299+0000 7f35cd61f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:48:40.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:40 smithi090 ceph-mon[40003]: pgmap v42: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:48:40.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:40 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:40.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:40 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:40.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:40 smithi090 ceph-mon[29100]: pgmap v42: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:48:40.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:40 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:40.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:40 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:40.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:40.639+0000 7f35cd61f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:48:41.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:40.987+0000 7f35cd61f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:48:41.880 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:41.520+0000 7f35cd61f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:48:41.882 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:41.882 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:41.882 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[29100]: pgmap v43: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-12T08:48:41.882 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:41.882 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:41.883 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2862113018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:42.134 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:41.913+0000 7f35cd61f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:48:42.135 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:42.135 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:42.135 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[40003]: pgmap v43: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-12T08:48:42.135 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:42.135 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:42.135 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:41 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2862113018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:41 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:41 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:41 smithi114 ceph-mon[33573]: pgmap v43: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-12T08:48:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:41 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:41 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:41 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2862113018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:42.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:42 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:42.317+0000 7f35cd61f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:48:43.558 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:48:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:48:43.507Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T08:48:43.558 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:48:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:48:43.509Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T08:48:43.812 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:43.656+0000 7f35cd61f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:48:43.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:43 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:48:43] "GET /metrics HTTP/1.1" 200 36195 "" "Prometheus/2.43.0" 2024-04-12T08:48:44.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:43 smithi090 ceph-mon[29100]: pgmap v44: 89 pgs: 89 active+clean; 1017 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-04-12T08:48:44.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:43 smithi090 ceph-mon[40003]: pgmap v44: 89 pgs: 89 active+clean; 1017 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-04-12T08:48:44.241 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:43 smithi114 ceph-mon[33573]: pgmap v44: 89 pgs: 89 active+clean; 1017 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-04-12T08:48:44.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:44.571+0000 7f35cd61f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:48:44.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:44.730+0000 7f35cd61f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:48:45.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:45 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3397898503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:45.328 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:45 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3397898503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:45.328 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:45.097+0000 7f35cd61f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:48:45.328 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:48:45] ENGINE Bus STARTING 2024-04-12T08:48:45.328 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: CherryPy Checker: 2024-04-12T08:48:45.328 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: The Application mounted at '' has an empty config. 2024-04-12T08:48:45.328 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:45.328 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:48:45] ENGINE Serving on http://:::9283 2024-04-12T08:48:45.328 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:45 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:48:45] ENGINE Bus STARTED 2024-04-12T08:48:45.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:45 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3397898503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[29100]: pgmap v45: 89 pgs: 89 active+clean; 1017 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2024-04-12T08:48:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[29100]: Standby manager daemon y restarted 2024-04-12T08:48:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[29100]: Standby manager daemon y started 2024-04-12T08:48:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T08:48:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:48:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T08:48:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:48:46.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[40003]: pgmap v45: 89 pgs: 89 active+clean; 1017 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2024-04-12T08:48:46.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[40003]: Standby manager daemon y restarted 2024-04-12T08:48:46.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[40003]: Standby manager daemon y started 2024-04-12T08:48:46.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T08:48:46.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:48:46.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T08:48:46.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:48:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-mon[33573]: pgmap v45: 89 pgs: 89 active+clean; 1017 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2024-04-12T08:48:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-mon[33573]: Standby manager daemon y restarted 2024-04-12T08:48:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-mon[33573]: Standby manager daemon y started 2024-04-12T08:48:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T08:48:46.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:48:46.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T08:48:46.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:48:46.665 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T08:48:46.381Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=4 2024-04-12T08:48:46.916 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:48:46] ENGINE Bus STOPPING 2024-04-12T08:48:46.916 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:48:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T08:48:46.916 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:48:46] ENGINE Bus STOPPED 2024-04-12T08:48:46.917 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:48:46] ENGINE Bus STARTING 2024-04-12T08:48:46.917 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:46.647+0000 7f701e39a700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-12T08:48:46.917 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ignoring --setuser ceph since I am not root 2024-04-12T08:48:46.917 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ignoring --setgroup ceph since I am not root 2024-04-12T08:48:46.917 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:46 smithi114 ceph-mgr[66925]: -- 172.21.15.114:0/2232822795 <== mon.1 v2:172.21.15.114:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5622f1a085a0 con 0x5622f19dcc00 2024-04-12T08:48:47.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:48:46] ENGINE Serving on http://:::9283 2024-04-12T08:48:47.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:48:46] ENGINE Bus STARTED 2024-04-12T08:48:47.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:47.025+0000 7f3530a81700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:48:47.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:47.025+0000 7f3530a81700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:48:47.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:47.025+0000 7f3530a81700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:48:47.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:47.025+0000 7f3530a81700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:48:47.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:47.025+0000 7f3530a81700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:48:47.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:47.033+0000 7f3529a73700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:48:47.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:47.033+0000 7f3529a73700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:48:47.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:47.033+0000 7f3529a73700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:48:47.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:47.033+0000 7f3529a73700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:48:47.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:48:47.033+0000 7f3529a73700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:48:47.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: mgrmap e27: x(active, since 85s), standbys: y 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: pgmap v46: 89 pgs: 89 active+clean; 1017 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: osdmap e77: 8 total, 8 up, 8 in 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.24802 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: mgrmap e28: y(active, starting, since 0.00738395s) 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:48:47.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:48:47.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: Manager daemon y is now available 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4127659483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: mgrmap e27: x(active, since 85s), standbys: y 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:47.177 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: pgmap v46: 89 pgs: 89 active+clean; 1017 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: osdmap e77: 8 total, 8 up, 8 in 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.24802 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: mgrmap e28: y(active, starting, since 0.00738395s) 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:48:47.178 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:48:47.179 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:48:47.180 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: Manager daemon y is now available 2024-04-12T08:48:47.180 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4127659483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:47.180 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:47.180 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:47.180 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:48:47.180 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:47 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:48:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: mgrmap e27: x(active, since 85s), standbys: y 2024-04-12T08:48:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:48:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:48:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: pgmap v46: 89 pgs: 89 active+clean; 1017 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2024-04-12T08:48:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.24802 172.21.15.114:0/3131549161' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: osdmap e77: 8 total, 8 up, 8 in 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.24802 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: mgrmap e28: y(active, starting, since 0.00738395s) 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:48:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: Manager daemon y is now available 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4127659483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:47.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:48:47.371 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:48:47.371 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:48:47.371 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:48:47.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:47.565+0000 7ff31a34d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:48:47.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:47 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:47.719+0000 7ff31a34d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:48:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:48 smithi114 ceph-mon[33573]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T08:48:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:48 smithi114 ceph-mon[33573]: mgrmap e29: y(active, since 1.0204s) 2024-04-12T08:48:48.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:48 smithi090 ceph-mon[40003]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T08:48:48.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:48 smithi090 ceph-mon[40003]: mgrmap e29: y(active, since 1.0204s) 2024-04-12T08:48:48.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:48 smithi090 ceph-mon[29100]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T08:48:48.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:48 smithi090 ceph-mon[29100]: mgrmap e29: y(active, since 1.0204s) 2024-04-12T08:48:48.849 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:48:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:48:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:48:48.849 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:48 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-12T08:48:48.849 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:48 smithi114 systemd[1]: Stopped Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:48.849 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:48 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Consumed 1.034s CPU time. 2024-04-12T08:48:48.849 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:48 smithi114 systemd[1]: Starting Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:48:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:49 smithi114 ceph-mon[33573]: [12/Apr/2024:08:48:48] ENGINE Bus STARTING 2024-04-12T08:48:49.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:49 smithi114 ceph-mon[33573]: [12/Apr/2024:08:48:48] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:48:49.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:49 smithi114 ceph-mon[33573]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T08:48:49.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:49 smithi114 ceph-mon[33573]: Cluster is now healthy 2024-04-12T08:48:49.118 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:48.847+0000 7ff31a34d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:48:49.118 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:49 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:49.007+0000 7ff31a34d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:48:49.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:49 smithi090 ceph-mon[40003]: [12/Apr/2024:08:48:48] ENGINE Bus STARTING 2024-04-12T08:48:49.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:49 smithi090 ceph-mon[40003]: [12/Apr/2024:08:48:48] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:48:49.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:49 smithi090 ceph-mon[40003]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T08:48:49.308 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:49 smithi090 ceph-mon[40003]: Cluster is now healthy 2024-04-12T08:48:49.309 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:49 smithi090 ceph-mon[29100]: [12/Apr/2024:08:48:48] ENGINE Bus STARTING 2024-04-12T08:48:49.309 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:49 smithi090 ceph-mon[29100]: [12/Apr/2024:08:48:48] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:48:49.309 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:49 smithi090 ceph-mon[29100]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T08:48:49.309 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:49 smithi090 ceph-mon[29100]: Cluster is now healthy 2024-04-12T08:48:49.527 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:49 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:49.247+0000 7ff31a34d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:48:49.528 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:49 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:49.390+0000 7ff31a34d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:48:49.528 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:49 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:49.523+0000 7ff31a34d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:48:49.867 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:49 smithi114 podman[77355]: 2024-04-12 08:48:49.600198573 +0000 UTC m=+0.374825736 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T08:48:50.054 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:50 smithi090 ceph-mon[29100]: pgmap v4: 89 pgs: 89 active+clean; 746 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-04-12T08:48:50.221 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:49 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:49.939+0000 7ff31a34d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:48:50.221 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:50 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:50.117+0000 7ff31a34d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:48:50.221 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:50 smithi114 podman[77355]: 2024-04-12 08:48:50.087894245 +0000 UTC m=+0.862521402 container create 84b339041c8215ac901cb928503709277d9d6609b2574ef7992d870d528da531 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, io.buildah.version=1.32.0, name=ubi8, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , version=8.5, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container) 2024-04-12T08:48:50.226 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:50 smithi114 ceph-mon[33573]: pgmap v4: 89 pgs: 89 active+clean; 746 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-04-12T08:48:50.226 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:50 smithi114 ceph-mon[33573]: [12/Apr/2024:08:48:48] ENGINE Serving on http://172.21.15.90:8765 2024-04-12T08:48:50.226 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:50 smithi114 ceph-mon[33573]: [12/Apr/2024:08:48:48] ENGINE Bus STARTED 2024-04-12T08:48:50.226 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:50 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3004960930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:50.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:50 smithi090 ceph-mon[40003]: pgmap v4: 89 pgs: 89 active+clean; 746 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-04-12T08:48:50.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:50 smithi090 ceph-mon[40003]: [12/Apr/2024:08:48:48] ENGINE Serving on http://172.21.15.90:8765 2024-04-12T08:48:50.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:50 smithi090 ceph-mon[40003]: [12/Apr/2024:08:48:48] ENGINE Bus STARTED 2024-04-12T08:48:50.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:50 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3004960930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:50.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:50 smithi090 ceph-mon[29100]: [12/Apr/2024:08:48:48] ENGINE Serving on http://172.21.15.90:8765 2024-04-12T08:48:50.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:50 smithi090 ceph-mon[29100]: [12/Apr/2024:08:48:48] ENGINE Bus STARTED 2024-04-12T08:48:50.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:50 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3004960930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:50.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:50 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:50.281+0000 7ff31a34d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:48:50.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:50 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:50.458+0000 7ff31a34d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:48:50.618 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:50 smithi114 podman[77355]: 2024-04-12 08:48:50.38296951 +0000 UTC m=+1.157596669 container init 84b339041c8215ac901cb928503709277d9d6609b2574ef7992d870d528da531 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, name=ubi8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.openshift.tags=base rhel8, io.openshift.expose-services=, distribution-scope=public, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-12T08:48:50.618 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:50 smithi114 podman[77355]: 2024-04-12 08:48:50.392483268 +0000 UTC m=+1.167110424 container start 84b339041c8215ac901cb928503709277d9d6609b2574ef7992d870d528da531 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.buildah.version=1.32.0, name=ubi8, build-date=2022-04-29T11:48:11.055897) 2024-04-12T08:48:50.618 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:50 smithi114 bash[77355]: 84b339041c8215ac901cb928503709277d9d6609b2574ef7992d870d528da531 2024-04-12T08:48:50.619 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:50 smithi114 systemd[1]: Started Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:48:51.075 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:50 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:50.763+0000 7ff31a34d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:48:51.075 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:50 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[77407]: logger=settings t=2024-04-12T08:48:50.744752902Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-12T08:48:51.075 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:50 smithi114 podman[77479]: 2024-04-12 08:48:50.808037719 +0000 UTC m=+0.041830158 container died 84b339041c8215ac901cb928503709277d9d6609b2574ef7992d870d528da531 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.component=ubi8-container, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, distribution-scope=public, vendor=Red Hat, Inc., io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, maintainer=Paul Cuzner ) 2024-04-12T08:48:51.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:51 smithi090 ceph-mon[29100]: mgrmap e30: y(active, since 3s) 2024-04-12T08:48:51.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:51 smithi090 ceph-mon[40003]: mgrmap e30: y(active, since 3s) 2024-04-12T08:48:51.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:51 smithi114 ceph-mon[33573]: mgrmap e30: y(active, since 3s) 2024-04-12T08:48:51.975 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:51 smithi114 podman[77479]: 2024-04-12 08:48:51.682642494 +0000 UTC m=+0.916434916 container remove 84b339041c8215ac901cb928503709277d9d6609b2574ef7992d870d528da531 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.openshift.tags=base rhel8, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.expose-services=, vcs-type=git, version=8.5, name=ubi8, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-12T08:48:51.975 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:51 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-12T08:48:52.171 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:52 smithi090 ceph-mon[40003]: pgmap v5: 89 pgs: 89 active+clean; 746 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-04-12T08:48:52.171 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:52 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2442235974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:52.171 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:52 smithi090 ceph-mon[29100]: pgmap v5: 89 pgs: 89 active+clean; 746 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-04-12T08:48:52.171 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:52 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2442235974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:52 smithi114 ceph-mon[33573]: pgmap v5: 89 pgs: 89 active+clean; 746 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-04-12T08:48:52.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:52 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2442235974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:52.681 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:52 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:52.387+0000 7ff31a34d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:48:53.087 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:52 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:52.606+0000 7ff31a34d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:48:53.367 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:53 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:48:53.367 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:48:53 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Consumed 1.092s CPU time. 2024-04-12T08:48:53.368 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:53 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:53.087+0000 7ff31a34d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:48:53.368 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:53 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:53.248+0000 7ff31a34d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:48:53.842 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:53 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:53.588+0000 7ff31a34d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:48:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:53 smithi114 ceph-mon[33573]: pgmap v6: 89 pgs: 89 active+clean; 732 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 45 op/s 2024-04-12T08:48:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:53 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:53 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:54.118 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:53 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:53.902+0000 7ff31a34d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:48:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:53 smithi090 ceph-mon[40003]: pgmap v6: 89 pgs: 89 active+clean; 732 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 45 op/s 2024-04-12T08:48:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:53 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:53 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:54.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:53 smithi090 ceph-mon[29100]: pgmap v6: 89 pgs: 89 active+clean; 732 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 45 op/s 2024-04-12T08:48:54.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:53 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:54.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:53 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:54.783 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:54 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:54.389+0000 7ff31a34d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:48:55.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:54 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:54.782+0000 7ff31a34d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:48:55.465 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:55 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:55.211+0000 7ff31a34d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:48:55.586 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:55 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.587 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:55 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.587 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:55 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.587 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:55 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.587 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:55 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/818904362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:55.587 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:55 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.587 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:55 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.587 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:55 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.587 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:55 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.587 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:55 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/818904362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:55.772 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:55 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.772 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:55 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.772 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:55 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.772 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:55 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:55.772 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:55 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/818904362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:56.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:56 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:56.701+0000 7ff31a34d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:48:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:56 smithi114 ceph-mon[33573]: pgmap v7: 89 pgs: 89 active+clean; 534 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 504 op/s 2024-04-12T08:48:56.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:56 smithi090 ceph-mon[29100]: pgmap v7: 89 pgs: 89 active+clean; 534 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 504 op/s 2024-04-12T08:48:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:56 smithi090 ceph-mon[40003]: pgmap v7: 89 pgs: 89 active+clean; 534 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 504 op/s 2024-04-12T08:48:57.820 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:57 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:57.820 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:57 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:57.820 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:57 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/301179143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:57.820 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:57 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:57.821 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:57 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:57.821 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:57 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/301179143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:57.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:57 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:57.556+0000 7ff31a34d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:48:57.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:57 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:57.722+0000 7ff31a34d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:48:57.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:57 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:57.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:57 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:48:57.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:57 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/301179143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:48:58.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:58.090+0000 7ff31a34d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:48:58.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:48:58] ENGINE Bus STARTING 2024-04-12T08:48:58.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: CherryPy Checker: 2024-04-12T08:48:58.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: The Application mounted at '' has an empty config. 2024-04-12T08:48:58.368 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: 2024-04-12T08:48:58.368 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:48:58] ENGINE Serving on http://:::9283 2024-04-12T08:48:58.368 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: [12/Apr/2024:08:48:58] ENGINE Bus STARTED 2024-04-12T08:48:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-mon[33573]: pgmap v8: 89 pgs: 89 active+clean; 534 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 391 op/s 2024-04-12T08:48:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-mon[33573]: Standby manager daemon x started 2024-04-12T08:48:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:48:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:48:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:48:58.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:58 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:48:58.911 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[29100]: pgmap v8: 89 pgs: 89 active+clean; 534 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 391 op/s 2024-04-12T08:48:58.911 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[29100]: Standby manager daemon x started 2024-04-12T08:48:58.912 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:48:58.912 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:48:58.912 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:48:58.912 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:48:58.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[40003]: pgmap v8: 89 pgs: 89 active+clean; 534 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 391 op/s 2024-04-12T08:48:58.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[40003]: Standby manager daemon x started 2024-04-12T08:48:58.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:48:58.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:48:58.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:48:58.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:58 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/233498221' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:48:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:59 smithi114 ceph-mon[33573]: mgrmap e31: y(active, since 11s), standbys: x 2024-04-12T08:48:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:48:59 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:48:59.903 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:59 smithi090 ceph-mon[40003]: mgrmap e31: y(active, since 11s), standbys: x 2024-04-12T08:48:59.903 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:48:59 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:48:59.903 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:59 smithi090 ceph-mon[29100]: mgrmap e31: y(active, since 11s), standbys: x 2024-04-12T08:48:59.903 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:48:59 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:49:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:00 smithi114 ceph-mon[33573]: pgmap v9: 89 pgs: 89 active+clean; 315 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 640 op/s 2024-04-12T08:49:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:00 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4054592801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:00 smithi090 ceph-mon[29100]: pgmap v9: 89 pgs: 89 active+clean; 315 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 640 op/s 2024-04-12T08:49:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:00 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4054592801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:00 smithi090 ceph-mon[40003]: pgmap v9: 89 pgs: 89 active+clean; 315 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 640 op/s 2024-04-12T08:49:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:00 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4054592801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:02 smithi114 ceph-mon[33573]: pgmap v10: 89 pgs: 89 active+clean; 315 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 584 op/s 2024-04-12T08:49:02.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:02 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:02.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:02 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:02.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:02 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:02.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:02 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:49:02.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:02 smithi114 ceph-mon[33573]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:49:02.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:02 smithi114 ceph-mon[33573]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:49:02.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:02 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[40003]: pgmap v10: 89 pgs: 89 active+clean; 315 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 584 op/s 2024-04-12T08:49:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:49:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[40003]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:49:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:49:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[29100]: pgmap v10: 89 pgs: 89 active+clean; 315 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 584 op/s 2024-04-12T08:49:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:49:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[29100]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:49:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[29100]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:49:02.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:02 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:03.688 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[66921]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T08:49:03.688 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:03 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-12T08:49:03.688 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:03 smithi114 systemd[1]: Stopped Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:03.689 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:03 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Consumed 1.092s CPU time. 2024-04-12T08:49:03.689 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:03 smithi114 systemd[1]: Starting Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:03.689 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-mon[33573]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:49:03.689 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-mon[33573]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:49:03.689 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-mon[33573]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:49:03.689 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-mon[33573]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:49:03.689 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/385686213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:03.689 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.689 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.689 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.689 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.689 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.902 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Removed 132948 objects 2024-04-12T08:49:03.902 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Clean up completed and total clean up time :164.735 2024-04-12T08:49:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:49:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[40003]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:49:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:49:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[40003]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:49:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/385686213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[29100]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:49:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[29100]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:49:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[29100]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:49:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[29100]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:49:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/385686213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:03.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:03 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:04.117 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:03 smithi114 podman[78828]: 2024-04-12 08:49:03.591901047 +0000 UTC m=+0.037365768 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T08:49:04.117 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:03 smithi114 podman[78828]: 2024-04-12 08:49:03.72861023 +0000 UTC m=+0.174075024 container create 4e3edc283f48adbbcb556e7d6634c1df2e121db59fcb9419c4285c80103beab4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, architecture=x86_64, vcs-type=git, distribution-scope=public, version=8.5, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.openshift.expose-services=) 2024-04-12T08:49:04.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:03.930Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.114:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.114:8765: connect: connection refused" 2024-04-12T08:49:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:03.931Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.114:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.114:8765: connect: connection refused" 2024-04-12T08:49:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:03.934Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.114:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.114:8765: connect: connection refused" 2024-04-12T08:49:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:03.935Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.114:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.114:8765: connect: connection refused" 2024-04-12T08:49:04.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:03 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:03.935Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.114:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.114:8765: connect: connection refused" 2024-04-12T08:49:04.354 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-04-12T08:49:04.355 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-04-12T08:49:04.400 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:04 smithi114 podman[78828]: 2024-04-12 08:49:04.265442791 +0000 UTC m=+0.710907497 container init 4e3edc283f48adbbcb556e7d6634c1df2e121db59fcb9419c4285c80103beab4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, description=Ceph Grafana Container, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.openshift.tags=base rhel8, vcs-type=git, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-12T08:49:04.400 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:04 smithi114 podman[78828]: 2024-04-12 08:49:04.26844434 +0000 UTC m=+0.713909047 container start 4e3edc283f48adbbcb556e7d6634c1df2e121db59fcb9419c4285c80103beab4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, name=ubi8, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, io.openshift.expose-services=, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, release=239.1651231664) 2024-04-12T08:49:04.400 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:04 smithi114 bash[78828]: 4e3edc283f48adbbcb556e7d6634c1df2e121db59fcb9419c4285c80103beab4 2024-04-12T08:49:04.400 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:04 smithi114 systemd[1]: Started Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:04.612 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:49:04.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:49:04.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[40003]: pgmap v11: 89 pgs: 89 active+clean; 315 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 584 op/s 2024-04-12T08:49:04.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[40003]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:49:04.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[40003]: pgmap v12: 89 pgs: 89 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 1.03k op/s 2024-04-12T08:49:04.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[40003]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T08:49:04.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[40003]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T08:49:04.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[29100]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:49:04.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[29100]: pgmap v11: 89 pgs: 89 active+clean; 315 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 584 op/s 2024-04-12T08:49:04.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[29100]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:49:04.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[29100]: pgmap v12: 89 pgs: 89 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 1.03k op/s 2024-04-12T08:49:04.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[29100]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T08:49:04.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:04 smithi090 ceph-mon[29100]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T08:49:04.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:04 smithi114 ceph-mon[33573]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:49:04.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:04 smithi114 ceph-mon[33573]: pgmap v11: 89 pgs: 89 active+clean; 315 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 584 op/s 2024-04-12T08:49:04.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:04 smithi114 ceph-mon[33573]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:49:04.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:04 smithi114 ceph-mon[33573]: pgmap v12: 89 pgs: 89 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 1.03k op/s 2024-04-12T08:49:04.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:04 smithi114 ceph-mon[33573]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T08:49:04.753 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:04 smithi114 ceph-mon[33573]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T08:49:04.753 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:04 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[78907]: logger=settings t=2024-04-12T08:49:04.422936896Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-12T08:49:04.754 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:04 smithi114 podman[78943]: 2024-04-12 08:49:04.451354566 +0000 UTC m=+0.020292902 container died 4e3edc283f48adbbcb556e7d6634c1df2e121db59fcb9419c4285c80103beab4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, name=ubi8, distribution-scope=public, io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container) 2024-04-12T08:49:05.367 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:05 smithi114 podman[78943]: 2024-04-12 08:49:05.094468571 +0000 UTC m=+0.663406907 container remove 4e3edc283f48adbbcb556e7d6634c1df2e121db59fcb9419c4285c80103beab4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.openshift.expose-services=, vcs-type=git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, version=8.5, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, release=239.1651231664, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, distribution-scope=public) 2024-04-12T08:49:05.367 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:05 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-12T08:49:05.708 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:05 smithi114 ceph-mon[33573]: Reconfiguring daemon prometheus.a on smithi114 2024-04-12T08:49:05.708 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:05 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/302702025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:05.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:05 smithi090 ceph-mon[40003]: Reconfiguring daemon prometheus.a on smithi114 2024-04-12T08:49:05.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:05 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/302702025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:05 smithi090 ceph-mon[29100]: Reconfiguring daemon prometheus.a on smithi114 2024-04-12T08:49:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:05 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/302702025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:06.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:06 smithi090 ceph-mon[40003]: pgmap v13: 89 pgs: 89 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-12T08:49:06.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:06 smithi090 ceph-mon[29100]: pgmap v13: 89 pgs: 89 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-12T08:49:06.738 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:06 smithi114 ceph-mon[33573]: pgmap v13: 89 pgs: 89 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-12T08:49:07.541 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'unique_pool_0' removed 2024-04-12T08:49:07.794 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:07 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1245770586' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:49:07.794 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:07 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:49:07.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:07 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1245770586' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:49:07.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:07 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:49:07.859 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:07 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1245770586' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:49:07.859 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:07 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:49:08.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 systemd[1]: Stopping Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:08.283 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-12T08:49:08.284 INFO:tasks.radosbench:Beginning radosbench... 2024-04-12T08:49:08.284 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-04-12T08:49:08.284 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-04-12T08:49:08.513 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:08 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.299Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-12T08:49:08.514 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[75655]: ts=2024-04-12T08:49:08.300Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-12T08:49:08.515 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 podman[79104]: 2024-04-12 08:49:08.309956451 +0000 UTC m=+0.093226796 container died 0765e8b7f635952396ab1620fcb9a055370307ee34a157b90d988de3cd906f73 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:49:08.548 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:49:08.671 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:08 smithi090 ceph-mon[40003]: pgmap v14: 89 pgs: 89 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-12T08:49:08.671 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:08 smithi090 ceph-mon[40003]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T08:49:08.671 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:08 smithi090 ceph-mon[40003]: osdmap e78: 8 total, 8 up, 8 in 2024-04-12T08:49:08.671 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:08 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4250292871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:08.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:08 smithi090 ceph-mon[29100]: pgmap v14: 89 pgs: 89 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-12T08:49:08.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:08 smithi090 ceph-mon[29100]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T08:49:08.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:08 smithi090 ceph-mon[29100]: osdmap e78: 8 total, 8 up, 8 in 2024-04-12T08:49:08.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:08 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4250292871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:08.851 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 podman[79104]: 2024-04-12 08:49:08.632957074 +0000 UTC m=+0.416227418 container cleanup 0765e8b7f635952396ab1620fcb9a055370307ee34a157b90d988de3cd906f73 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:49:08.851 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 bash[79104]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a 2024-04-12T08:49:08.851 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-mon[33573]: pgmap v14: 89 pgs: 89 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-12T08:49:08.851 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-mon[33573]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T08:49:08.851 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-mon[33573]: osdmap e78: 8 total, 8 up, 8 in 2024-04-12T08:49:08.851 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:08 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4250292871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:09.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:08 smithi114 podman[79118]: 2024-04-12 08:49:08.850938147 +0000 UTC m=+0.539289370 container remove 0765e8b7f635952396ab1620fcb9a055370307ee34a157b90d988de3cd906f73 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:49:09.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-12T08:49:09.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 systemd[1]: Stopped Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:09.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@prometheus.a.service: Consumed 1.241s CPU time. 2024-04-12T08:49:09.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 systemd[1]: Starting Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:09.596 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 podman[79220]: 2024-04-12 08:49:09.210828334 +0000 UTC m=+0.019026749 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-12T08:49:09.596 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 podman[79220]: 2024-04-12 08:49:09.335200848 +0000 UTC m=+0.143399234 container create f009118ca428198f620c5a61aa9bcdce3a3e72d7431061e66f0272b89a93ac8c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 podman[79220]: 2024-04-12 08:49:09.595925617 +0000 UTC m=+0.404124001 container init f009118ca428198f620c5a61aa9bcdce3a3e72d7431061e66f0272b89a93ac8c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 podman[79220]: 2024-04-12 08:49:09.59858147 +0000 UTC m=+0.406779855 container start f009118ca428198f620c5a61aa9bcdce3a3e72d7431061e66f0272b89a93ac8c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.621Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.621Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.621Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi114 (none))" 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.621Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.621Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.623Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.624Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.625Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-12T08:49:09.868 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.625Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.626Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.626Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.975µs 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.626Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.626Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.645Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.647Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.647Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.647Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=26.503µs wal_replay_duration=21.302077ms wbl_replay_duration=138ns total_replay_duration=21.370462ms 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.650Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.650Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.650Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.665Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.570795ms db_storage=760ns remote_storage=1.212µs web_handler=433ns query_engine=648ns scrape=201.573µs scrape_sd=95.22µs notify=10.56µs notify_sd=25.381µs rules=14.950105ms tracing=5.078µs 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.665Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:09.665Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-12T08:49:09.869 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 bash[79220]: f009118ca428198f620c5a61aa9bcdce3a3e72d7431061e66f0272b89a93ac8c 2024-04-12T08:49:09.870 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:09 smithi114 systemd[1]: Started Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:10.763 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'unique_pool_1' created 2024-04-12T08:49:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:10 smithi114 ceph-mon[33573]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:10 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:10 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:10 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:49:10.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:10 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:49:10.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:10 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3278282463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:11.054 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[40003]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:11.054 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:11.054 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:11.054 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:49:11.054 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:49:11.054 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3278282463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:11.055 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[29100]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:11.055 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:11.055 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:11.055 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:49:11.055 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:49:11.055 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:10 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3278282463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:11.459 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-04-12T08:49:11.719 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:49:11.977 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[40003]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:49:11.977 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1018803354' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-12T08:49:11.977 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1018803354' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-12T08:49:11.977 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[40003]: osdmap e79: 8 total, 8 up, 8 in 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[40003]: pgmap v18: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[40003]: Upgrade: Updating mgr.x 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[40003]: Deploying daemon mgr.x on smithi114 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[29100]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1018803354' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1018803354' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[29100]: osdmap e79: 8 total, 8 up, 8 in 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[29100]: pgmap v18: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:11.978 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[29100]: Upgrade: Updating mgr.x 2024-04-12T08:49:11.979 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:49:11.979 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:49:11.979 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:11.979 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:11 smithi090 ceph-mon[29100]: Deploying daemon mgr.x on smithi114 2024-04-12T08:49:12.021 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:11 smithi114 ceph-mon[33573]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T08:49:12.021 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:11 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1018803354' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-12T08:49:12.021 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:11 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1018803354' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-12T08:49:12.021 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:11 smithi114 ceph-mon[33573]: osdmap e79: 8 total, 8 up, 8 in 2024-04-12T08:49:12.021 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:11 smithi114 ceph-mon[33573]: pgmap v18: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:12.021 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:11 smithi114 ceph-mon[33573]: Upgrade: Updating mgr.x 2024-04-12T08:49:12.021 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:11 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:49:12.021 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:11 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:49:12.021 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:11 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:12.021 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:11 smithi114 ceph-mon[33573]: Deploying daemon mgr.x on smithi114 2024-04-12T08:49:12.847 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:12 smithi114 ceph-mon[33573]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:49:12.847 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:12 smithi114 ceph-mon[33573]: osdmap e80: 8 total, 8 up, 8 in 2024-04-12T08:49:12.847 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:12 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3888305207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:13.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:12 smithi090 ceph-mon[40003]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:49:13.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:12 smithi090 ceph-mon[40003]: osdmap e80: 8 total, 8 up, 8 in 2024-04-12T08:49:13.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:12 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3888305207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:12 smithi090 ceph-mon[29100]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:49:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:12 smithi090 ceph-mon[29100]: osdmap e80: 8 total, 8 up, 8 in 2024-04-12T08:49:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:12 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3888305207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:13.416 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:13 smithi114 systemd[1]: Stopping Ceph mgr.x for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:13.753 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:13 smithi114 podman[79555]: 2024-04-12 08:49:13.415451762 +0000 UTC m=+0.134611789 container died 8e82f0405773258ab496cfaaf2e8b3893ac4ddb81e4c2c3610dff2f26f6ce37f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-12T08:49:14.036 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:13 smithi114 ceph-mon[33573]: osdmap e81: 8 total, 8 up, 8 in 2024-04-12T08:49:14.037 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:13 smithi114 ceph-mon[33573]: pgmap v21: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:14.037 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:13 smithi114 podman[79555]: 2024-04-12 08:49:13.851951602 +0000 UTC m=+0.571111671 container cleanup 8e82f0405773258ab496cfaaf2e8b3893ac4ddb81e4c2c3610dff2f26f6ce37f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_CLEAN=True) 2024-04-12T08:49:14.037 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:13 smithi114 bash[79555]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x 2024-04-12T08:49:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:13 smithi090 ceph-mon[40003]: osdmap e81: 8 total, 8 up, 8 in 2024-04-12T08:49:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:13 smithi090 ceph-mon[40003]: pgmap v21: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:13 smithi090 ceph-mon[29100]: osdmap e81: 8 total, 8 up, 8 in 2024-04-12T08:49:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:13 smithi090 ceph-mon[29100]: pgmap v21: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:14.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 podman[79570]: 2024-04-12 08:49:14.03660662 +0000 UTC m=+0.625390303 container remove 8e82f0405773258ab496cfaaf2e8b3893ac4ddb81e4c2c3610dff2f26f6ce37f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-12T08:49:14.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-12T08:49:14.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-12T08:49:14.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 systemd[1]: Stopped Ceph mgr.x for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:14.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.x.service: Consumed 39.647s CPU time. 2024-04-12T08:49:14.769 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 systemd[1]: Starting Ceph mgr.x for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:14.769 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 podman[79702]: 2024-04-12 08:49:14.574746865 +0000 UTC m=+0.017841875 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:49:14.769 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 podman[79702]: 2024-04-12 08:49:14.696032145 +0000 UTC m=+0.139127186 container create 60c0ca408f9165df9aa655205a64edb0461d86d9a2d1f9e1daa87c85c71ae8f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-12T08:49:14.783 INFO:teuthology.orchestra.run.smithi090.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-04-12T08:49:15.064 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:14 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1503418543' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:49:15.065 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 podman[79702]: 2024-04-12 08:49:14.906974451 +0000 UTC m=+0.350069446 container init 60c0ca408f9165df9aa655205a64edb0461d86d9a2d1f9e1daa87c85c71ae8f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-12T08:49:15.065 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 podman[79702]: 2024-04-12 08:49:14.909814915 +0000 UTC m=+0.352909919 container start 60c0ca408f9165df9aa655205a64edb0461d86d9a2d1f9e1daa87c85c71ae8f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-12T08:49:15.065 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 ceph-mgr[79721]: -- 172.21.15.114:0/1753565604 <== mon.1 v2:172.21.15.114:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5640c9f605a0 con 0x5640c9f34800 2024-04-12T08:49:15.065 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 bash[79702]: 60c0ca408f9165df9aa655205a64edb0461d86d9a2d1f9e1daa87c85c71ae8f6 2024-04-12T08:49:15.065 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:14 smithi114 systemd[1]: Started Ceph mgr.x for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:15.161 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:14 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1503418543' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:49:15.162 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:14 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1503418543' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:49:15.777 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:15.477+0000 7f5a5fcf9200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:49:15.777 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:15.580+0000 7f5a5fcf9200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:49:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1503418543' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:49:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[40003]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T08:49:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1330086257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:49:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[40003]: pgmap v23: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:15.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1503418543' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:49:15.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[29100]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T08:49:15.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1330086257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:15.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:15.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:15.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:49:15.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:15 smithi090 ceph-mon[29100]: pgmap v23: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:16.096 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:15 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1503418543' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:49:16.096 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:15 smithi114 ceph-mon[33573]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T08:49:16.096 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:15 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1330086257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:16.096 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:15 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:16.096 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:15 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:16.096 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:15 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:49:16.096 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:15 smithi114 ceph-mon[33573]: pgmap v23: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:16.361 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:16 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:16.267+0000 7f5a5fcf9200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:49:16.361 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:16 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:16.360+0000 7f5a5fcf9200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:49:16.573 DEBUG:teuthology.orchestra.run.smithi090:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-04-12T08:49:16.577 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-12T08:49:16.577 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-12T08:49:16.613 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:16 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:16.515+0000 7f5a5fcf9200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:49:16.613 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:16 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:16.613+0000 7f5a5fcf9200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:49:16.627 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:hints = 1 2024-04-12T08:49:16.627 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-12T08:49:16.627 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Object prefix: benchmark_data_smithi090_95020 2024-04-12T08:49:16.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:49:16.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 0 0 0 0 0 0 - 0 2024-04-12T08:49:16.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:16 smithi114 ceph-mon[33573]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:49:16.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:16 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:16.758+0000 7f5a5fcf9200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:49:17.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:16 smithi090 ceph-mon[40003]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:49:17.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:16 smithi090 ceph-mon[29100]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:49:17.576 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:17 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:17.197+0000 7f5a5fcf9200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:49:17.576 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:17 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:17.412+0000 7f5a5fcf9200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:49:17.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 1 16 3668 3652 228.237 228.25 0.00471535 0.00436822 2024-04-12T08:49:17.826 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:17 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:17.826 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:17 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:17.575+0000 7f5a5fcf9200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:49:17.826 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:17 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:17.746+0000 7f5a5fcf9200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:49:18.075 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:17 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:18.075 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:17 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:18.075 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:17 smithi090 ceph-mon[40003]: pgmap v24: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:18.075 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:17 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3126046219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:18.075 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:17 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:18.075 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:17 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:18.075 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:17 smithi090 ceph-mon[29100]: pgmap v24: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:18.075 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:17 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3126046219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:17 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:17 smithi114 ceph-mon[33573]: pgmap v24: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:49:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:17 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3126046219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:18.118 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:18.021+0000 7f5a5fcf9200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:49:18.368 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:18 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-12T08:49:18.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 2 16 7122 7106 222.046 215.875 0.00239935 0.00447249 2024-04-12T08:49:18.867 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:18 smithi114 systemd[1]: Stopped Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:18.867 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:18 smithi114 systemd[1]: Starting Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:19.260 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:19 smithi114 podman[80223]: 2024-04-12 08:49:18.996357357 +0000 UTC m=+0.216071331 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T08:49:19.617 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:19 smithi114 podman[80223]: 2024-04-12 08:49:19.260168594 +0000 UTC m=+0.479882566 container create bcf41ef25c38f9e8c2e3fd047107fbdede68a6c5fc6358d71d56836914e48491 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, vcs-type=git, io.openshift.expose-services=, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , vendor=Red Hat, Inc., description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, architecture=x86_64, version=8.5, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-12T08:49:19.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 3 16 10257 10241 213.338 195.938 0.00318568 0.00467978 2024-04-12T08:49:19.958 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:19.681+0000 7f5a5fcf9200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:49:19.958 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:19.927+0000 7f5a5fcf9200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:49:19.959 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:19 smithi114 podman[80223]: 2024-04-12 08:49:19.841167341 +0000 UTC m=+1.060881322 container init bcf41ef25c38f9e8c2e3fd047107fbdede68a6c5fc6358d71d56836914e48491 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, release=239.1651231664, version=8.5, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, description=Ceph Grafana Container, maintainer=Paul Cuzner , vcs-type=git, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base rhel8) 2024-04-12T08:49:19.959 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:19 smithi114 podman[80223]: 2024-04-12 08:49:19.846055671 +0000 UTC m=+1.065769640 container start bcf41ef25c38f9e8c2e3fd047107fbdede68a6c5fc6358d71d56836914e48491 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, release=239.1651231664, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, maintainer=Paul Cuzner , name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5) 2024-04-12T08:49:20.286 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:19 smithi114 bash[80223]: bcf41ef25c38f9e8c2e3fd047107fbdede68a6c5fc6358d71d56836914e48491 2024-04-12T08:49:20.286 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:19 smithi114 systemd[1]: Started Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:20.286 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[80277]: logger=settings t=2024-04-12T08:49:20.144062652Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-12T08:49:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:20 smithi114 ceph-mon[33573]: pgmap v25: 89 pgs: 89 active+clean; 248 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s wr, 528 op/s 2024-04-12T08:49:20.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:20.484+0000 7f5a5fcf9200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:49:20.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 4 16 13232 13216 206.484 185.938 0.00540356 0.00483679 2024-04-12T08:49:20.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:20 smithi090 ceph-mon[29100]: pgmap v25: 89 pgs: 89 active+clean; 248 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s wr, 528 op/s 2024-04-12T08:49:20.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:20 smithi090 ceph-mon[40003]: pgmap v25: 89 pgs: 89 active+clean; 248 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s wr, 528 op/s 2024-04-12T08:49:20.974 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:20.642+0000 7f5a5fcf9200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:49:21.300 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:20.971+0000 7f5a5fcf9200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:49:21.300 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:21.297+0000 7f5a5fcf9200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:49:21.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:21 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1299532932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:21.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 5 15 16295 16280 203.483 191.5 0.00577994 0.00490849 2024-04-12T08:49:21.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:21 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/1299532932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:21.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:21 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1299532932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:22.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:21.800+0000 7f5a5fcf9200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:49:22.117 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:21 smithi114 podman[80319]: 2024-04-12 08:49:21.759068082 +0000 UTC m=+1.599998761 container remove bcf41ef25c38f9e8c2e3fd047107fbdede68a6c5fc6358d71d56836914e48491 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, release=239.1651231664, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, name=ubi8, version=8.5, com.redhat.component=ubi8-container, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.expose-services=, distribution-scope=public, io.buildah.version=1.32.0) 2024-04-12T08:49:22.118 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:21 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-12T08:49:22.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:22 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:22.226+0000 7f5a5fcf9200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:49:22.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:22 smithi114 ceph-mon[33573]: pgmap v26: 89 pgs: 89 active+clean; 248 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s wr, 466 op/s 2024-04-12T08:49:22.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:22 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:22.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:22 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:22.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:22 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:22.618 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:22 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:49:22.618 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:22 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Consumed 1.108s CPU time. 2024-04-12T08:49:22.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 6 16 19008 18992 197.817 169.5 0.0044867 0.00504984 2024-04-12T08:49:22.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:22 smithi090 ceph-mon[29100]: pgmap v26: 89 pgs: 89 active+clean; 248 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s wr, 466 op/s 2024-04-12T08:49:22.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:22 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:22.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:22 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:22.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:22 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:22.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:22 smithi090 ceph-mon[40003]: pgmap v26: 89 pgs: 89 active+clean; 248 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s wr, 466 op/s 2024-04-12T08:49:22.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:22 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:22.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:22 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:22.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:22 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:23.118 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:22 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:22.662+0000 7f5a5fcf9200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:49:23.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:23 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4156588146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:23.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 7 16 21561 21545 192.35 159.562 0.00364338 0.00519229 2024-04-12T08:49:23.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:23 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4156588146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:23.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:49:23] "GET /metrics HTTP/1.1" 200 36215 "" "Prometheus/2.43.0" 2024-04-12T08:49:23.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:23 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/4156588146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:24.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:24 smithi114 ceph-mon[33573]: pgmap v27: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-04-12T08:49:24.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:24 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:24.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:24 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:24.619 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:24.152+0000 7f5a5fcf9200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:49:24.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 8 16 24124 24108 188.328 160.188 0.00762936 0.00530464 2024-04-12T08:49:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:24 smithi090 ceph-mon[40003]: pgmap v27: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-04-12T08:49:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:24 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:24 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:24.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:24 smithi090 ceph-mon[29100]: pgmap v27: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-04-12T08:49:24.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:24 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:24.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:24 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:25.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:25.127+0000 7f5a5fcf9200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:49:25.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:25.297+0000 7f5a5fcf9200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:49:25.628 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 9 16 26488 26472 183.817 147.75 0.00167483 0.00542032 2024-04-12T08:49:25.911 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:25.626+0000 7f5a5fcf9200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:49:25.911 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: [12/Apr/2024:08:49:25] ENGINE Bus STARTING 2024-04-12T08:49:25.911 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: CherryPy Checker: 2024-04-12T08:49:25.911 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: The Application mounted at '' has an empty config. 2024-04-12T08:49:25.911 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:25.911 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: [12/Apr/2024:08:49:25] ENGINE Serving on http://:::9283 2024-04-12T08:49:25.911 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: [12/Apr/2024:08:49:25] ENGINE Bus STARTED 2024-04-12T08:49:26.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:26 smithi114 ceph-mon[33573]: pgmap v28: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-04-12T08:49:26.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:26 smithi114 ceph-mon[33573]: Standby manager daemon x restarted 2024-04-12T08:49:26.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:26 smithi114 ceph-mon[33573]: Standby manager daemon x started 2024-04-12T08:49:26.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:26 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:49:26.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:26 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:49:26.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:26 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:49:26.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:26 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:49:26.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:26 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2302631374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:26.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 10 16 28565 28549 178.416 129.812 0.00857129 0.00559954 2024-04-12T08:49:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[40003]: pgmap v28: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-04-12T08:49:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[40003]: Standby manager daemon x restarted 2024-04-12T08:49:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[40003]: Standby manager daemon x started 2024-04-12T08:49:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:49:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:49:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:49:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:49:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2302631374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:26.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[29100]: pgmap v28: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-04-12T08:49:26.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[29100]: Standby manager daemon x restarted 2024-04-12T08:49:26.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[29100]: Standby manager daemon x started 2024-04-12T08:49:26.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:49:26.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:49:26.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:49:26.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[29100]: from='mgr.? 172.21.15.114:0/4152658032' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:49:26.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:26 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/2302631374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:27.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:27 smithi114 ceph-mon[33573]: mgrmap e32: y(active, since 39s), standbys: x 2024-04-12T08:49:27.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 11 16 30975 30959 175.888 150.625 0.00417308 0.00567993 2024-04-12T08:49:27.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:27 smithi090 ceph-mon[29100]: mgrmap e32: y(active, since 39s), standbys: x 2024-04-12T08:49:27.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:27 smithi090 ceph-mon[40003]: mgrmap e32: y(active, since 39s), standbys: x 2024-04-12T08:49:28.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 12 16 33072 33056 172.152 131.062 0.00705803 0.00580336 2024-04-12T08:49:28.661 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:28 smithi114 ceph-mon[33573]: pgmap v29: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-12T08:49:28.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:28 smithi090 ceph-mon[29100]: pgmap v29: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-12T08:49:28.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:28 smithi090 ceph-mon[40003]: pgmap v29: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-12T08:49:29.023 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:49:29.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 13 16 35336 35320 169.793 141.5 0.00975959 0.00588472 2024-04-12T08:49:29.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:29 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/515530489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:29.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:29 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/515530489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:29.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:29 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/515530489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:30.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 14 16 37238 37222 166.155 118.875 0.00246607 0.00600484 2024-04-12T08:49:30.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: pgmap v30: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 162 MiB/s wr, 2.60k op/s 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: pgmap v31: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 165 MiB/s wr, 2.65k op/s 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: pgmap v32: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 206 MiB/s wr, 3.29k op/s 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-12T08:49:30.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-12T08:49:30.872 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-12T08:49:30.872 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T08:49:30.872 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:30 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-12T08:49:30.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: pgmap v30: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 162 MiB/s wr, 2.60k op/s 2024-04-12T08:49:30.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:30.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:49:30.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: pgmap v31: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 165 MiB/s wr, 2.65k op/s 2024-04-12T08:49:30.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: pgmap v32: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 206 MiB/s wr, 3.29k op/s 2024-04-12T08:49:30.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:49:30.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:49:30.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-12T08:49:30.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-12T08:49:30.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-12T08:49:30.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-12T08:49:30.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T08:49:30.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: pgmap v30: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 162 MiB/s wr, 2.60k op/s 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: pgmap v31: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 165 MiB/s wr, 2.65k op/s 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: pgmap v32: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 206 MiB/s wr, 3.29k op/s 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-12T08:49:30.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-12T08:49:30.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-12T08:49:30.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T08:49:30.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:30 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-12T08:49:31.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 15 16 38508 38492 160.369 79.375 0.00241966 0.00622991 2024-04-12T08:49:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:31 smithi114 ceph-mon[33573]: Upgrade: Setting container_image for all mgr 2024-04-12T08:49:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:31 smithi114 ceph-mon[33573]: Upgrade: It appears safe to stop mon.a 2024-04-12T08:49:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:31 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/691617182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:31.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:31 smithi090 ceph-mon[40003]: Upgrade: Setting container_image for all mgr 2024-04-12T08:49:31.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:31 smithi090 ceph-mon[40003]: Upgrade: It appears safe to stop mon.a 2024-04-12T08:49:31.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:31 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/691617182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:31 smithi090 ceph-mon[29100]: Upgrade: Setting container_image for all mgr 2024-04-12T08:49:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:31 smithi090 ceph-mon[29100]: Upgrade: It appears safe to stop mon.a 2024-04-12T08:49:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:31 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/691617182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:32.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 16 16 40148 40132 156.752 102.5 0.00815236 0.00637479 2024-04-12T08:49:32.676 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:32 smithi114 ceph-mon[33573]: pgmap v33: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-12T08:49:32.676 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:32 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:32.677 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:32 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-12T08:49:32.677 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:32 smithi114 systemd[1]: Stopped Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:32.677 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:32 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Consumed 1.108s CPU time. 2024-04-12T08:49:32.677 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:32 smithi114 systemd[1]: Starting Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:32.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:32 smithi090 ceph-mon[40003]: pgmap v33: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-12T08:49:32.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:32 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:32.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:32 smithi090 ceph-mon[29100]: pgmap v33: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-12T08:49:32.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:32 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:32.970 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:32 smithi114 podman[83249]: 2024-04-12 08:49:32.579078158 +0000 UTC m=+0.031877625 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T08:49:32.970 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:32 smithi114 podman[83249]: 2024-04-12 08:49:32.716581381 +0000 UTC m=+0.169380824 container create ee0b0484aa65403b77af4cafa957131caccaf700c9355f6a6d587cc004f8d3bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, maintainer=Paul Cuzner , vcs-type=git, release=239.1651231664, architecture=x86_64, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, description=Ceph Grafana Container, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container) 2024-04-12T08:49:32.970 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:32 smithi114 podman[83249]: 2024-04-12 08:49:32.970012981 +0000 UTC m=+0.422812423 container init ee0b0484aa65403b77af4cafa957131caccaf700c9355f6a6d587cc004f8d3bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, io.openshift.expose-services=, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, release=239.1651231664, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5) 2024-04-12T08:49:33.367 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:32 smithi114 podman[83249]: 2024-04-12 08:49:32.973322434 +0000 UTC m=+0.426121876 container start ee0b0484aa65403b77af4cafa957131caccaf700c9355f6a6d587cc004f8d3bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, vcs-type=git, release=239.1651231664, io.openshift.tags=base rhel8, distribution-scope=public, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, maintainer=Paul Cuzner , vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, com.redhat.component=ubi8-container) 2024-04-12T08:49:33.368 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:33 smithi114 bash[83249]: ee0b0484aa65403b77af4cafa957131caccaf700c9355f6a6d587cc004f8d3bc 2024-04-12T08:49:33.368 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:33 smithi114 systemd[1]: Started Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:33.368 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:33 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[83262]: logger=settings t=2024-04-12T08:49:33.147755892Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-12T08:49:33.368 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:33 smithi114 podman[83279]: 2024-04-12 08:49:33.172521858 +0000 UTC m=+0.018148962 container died ee0b0484aa65403b77af4cafa957131caccaf700c9355f6a6d587cc004f8d3bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, release=239.1651231664, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.buildah.version=1.32.0, version=8.5, name=ubi8) 2024-04-12T08:49:33.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 17 16 41367 41351 152.012 76.1875 0.00181744 0.00653823 2024-04-12T08:49:33.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:49:33] "GET /metrics HTTP/1.1" 200 36271 "" "Prometheus/2.43.0" 2024-04-12T08:49:34.085 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:33 smithi114 podman[83279]: 2024-04-12 08:49:33.791655333 +0000 UTC m=+0.637282456 container remove ee0b0484aa65403b77af4cafa957131caccaf700c9355f6a6d587cc004f8d3bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, com.redhat.component=ubi8-container, name=ubi8, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, description=Ceph Grafana Container, version=8.5, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, maintainer=Paul Cuzner , release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=) 2024-04-12T08:49:34.085 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:33 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-12T08:49:34.367 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:34 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:49:34.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 18 16 42870 42854 148.785 93.9375 0.00428004 0.00671072 2024-04-12T08:49:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:34 smithi114 ceph-mon[33573]: pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 160 MiB/s wr, 2.55k op/s 2024-04-12T08:49:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:34 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3678609319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:34.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:34 smithi090 ceph-mon[40003]: pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 160 MiB/s wr, 2.55k op/s 2024-04-12T08:49:34.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:34 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3678609319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:34.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:34 smithi090 ceph-mon[29100]: pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 160 MiB/s wr, 2.55k op/s 2024-04-12T08:49:34.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:34 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3678609319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:35.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 19 16 44341 44325 145.793 91.9375 0.0101983 0.00685424 2024-04-12T08:49:36.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:49:36.630108+0000 min lat: 0.00128221 max lat: 0.20158 avg lat: 0.00701356 2024-04-12T08:49:36.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:49:36.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 20 16 45612 45596 142.475 79.4375 0.00440344 0.00701356 2024-04-12T08:49:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:36 smithi114 ceph-mon[33573]: pgmap v35: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 160 MiB/s wr, 2.55k op/s 2024-04-12T08:49:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:36 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3309243736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:36 smithi090 ceph-mon[40003]: pgmap v35: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 160 MiB/s wr, 2.55k op/s 2024-04-12T08:49:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:36 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3309243736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:36.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:36 smithi090 ceph-mon[29100]: pgmap v35: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 160 MiB/s wr, 2.55k op/s 2024-04-12T08:49:36.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:36 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/3309243736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:37.629 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 21 16 47565 47549 141.502 122.062 0.00796252 0.00706162 2024-04-12T08:49:38.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 22 16 49199 49183 139.712 102.125 0.00678924 0.00715268 2024-04-12T08:49:38.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:49:38.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:38 smithi114 ceph-mon[33573]: pgmap v36: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-12T08:49:38.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:38 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:38.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:38 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:49:38.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:38 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:49:38.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:38 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:38.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:38 smithi090 ceph-mon[40003]: pgmap v36: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-12T08:49:38.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:38 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:38.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:38 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:49:38.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:38 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:49:38.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:38 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:38.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:38 smithi090 ceph-mon[29100]: pgmap v36: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-12T08:49:38.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:38 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' 2024-04-12T08:49:38.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:38 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:49:38.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:38 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:49:38.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:38 smithi090 ceph-mon[29100]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:49:39.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 23 16 50918 50902 138.308 107.438 0.017576 0.00722445 2024-04-12T08:49:39.866 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:39 smithi090 ceph-mon[29100]: Upgrade: Updating mon.a 2024-04-12T08:49:39.866 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:39 smithi090 ceph-mon[29100]: Deploying daemon mon.a on smithi090 2024-04-12T08:49:39.866 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:39 smithi090 ceph-mon[29100]: from='client.? 172.21.15.90:0/707761854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:39.866 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:39 smithi090 ceph-mon[40003]: Upgrade: Updating mon.a 2024-04-12T08:49:39.866 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:39 smithi090 ceph-mon[40003]: Deploying daemon mon.a on smithi090 2024-04-12T08:49:39.867 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:39 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/707761854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:39.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:39 smithi114 ceph-mon[33573]: Upgrade: Updating mon.a 2024-04-12T08:49:39.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:39 smithi114 ceph-mon[33573]: Deploying daemon mon.a on smithi090 2024-04-12T08:49:39.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:39 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/707761854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:40.620 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:40 smithi090 systemd[1]: Stopping Ceph mon.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:40.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 24 16 52624 52608 136.988 106.625 0.0123782 0.00729486 2024-04-12T08:49:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:40 smithi090 ceph-mon[40003]: pgmap v37: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 103 MiB/s wr, 1.66k op/s 2024-04-12T08:49:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:40 smithi090 ceph-mon[29100]: pgmap v37: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 103 MiB/s wr, 1.66k op/s 2024-04-12T08:49:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a[29096]: 2024-04-12T08:49:40.719+0000 7fdb6e8f3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T08:49:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a[29096]: 2024-04-12T08:49:40.719+0000 7fdb6e8f3700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-12T08:49:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:40 smithi090 podman[95608]: 2024-04-12 08:49:40.812810083 +0000 UTC m=+0.254906706 container died 5c90fd1861be8cb014276016d32819c28aa4c3b8c34bdd98f0a066b4905146d6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-12T08:49:40.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:49:40] ENGINE Bus STOPPING 2024-04-12T08:49:40.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:49:40] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T08:49:40.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:49:40] ENGINE Bus STOPPED 2024-04-12T08:49:40.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:49:40] ENGINE Bus STARTING 2024-04-12T08:49:40.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:49:40] ENGINE Serving on http://:::9283 2024-04-12T08:49:40.924 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:49:40] ENGINE Bus STARTED 2024-04-12T08:49:41.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:40 smithi114 ceph-mon[33573]: pgmap v37: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 103 MiB/s wr, 1.66k op/s 2024-04-12T08:49:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:41 smithi090 podman[95608]: 2024-04-12 08:49:41.31727287 +0000 UTC m=+0.759369498 container cleanup 5c90fd1861be8cb014276016d32819c28aa4c3b8c34bdd98f0a066b4905146d6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, RELEASE=quincy-d3b5060, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409) 2024-04-12T08:49:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:41 smithi090 bash[95608]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a 2024-04-12T08:49:41.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 25 16 54168 54152 135.368 96.5 0.0063164 0.00736683 2024-04-12T08:49:42.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 26 16 55290 55274 132.858 70.125 0.00596749 0.00752206 2024-04-12T08:49:43.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 27 16 56436 56420 130.59 71.625 0.00819564 0.00765239 2024-04-12T08:49:43.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:49:43] "GET /metrics HTTP/1.1" 200 36271 "" "Prometheus/2.43.0" 2024-04-12T08:49:44.323 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:44 smithi090 podman[95633]: 2024-04-12 08:49:44.070598603 +0000 UTC m=+3.252121349 container remove 5c90fd1861be8cb014276016d32819c28aa4c3b8c34bdd98f0a066b4905146d6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, RELEASE=quincy-d3b5060, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-12T08:49:44.393 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:44 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-04-12T08:49:44.393 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:44 smithi114 systemd[1]: Stopped Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:44.393 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:44 smithi114 systemd[1]: Starting Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:44.631 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 28 16 57895 57879 129.183 91.1875 0.0110275 0.00773529 2024-04-12T08:49:44.646 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:44 smithi114 podman[83362]: 2024-04-12 08:49:44.295421442 +0000 UTC m=+0.025503694 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T08:49:44.647 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:44 smithi114 podman[83362]: 2024-04-12 08:49:44.418166888 +0000 UTC m=+0.148249170 container create a5b220c30c3fc6c5b363ae3a6fe152b1c03fcea08c515cf80ec1a40feb3f5eae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.buildah.version=1.32.0, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, distribution-scope=public, vcs-type=git, vendor=Red Hat, Inc.) 2024-04-12T08:49:44.650 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:44 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-12T08:49:44.650 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:44 smithi090 systemd[1]: Stopped Ceph mon.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:44.650 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:44 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.a.service: Consumed 15.027s CPU time. 2024-04-12T08:49:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:44 smithi090 systemd[1]: Starting Ceph mon.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:44.948 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:44 smithi114 podman[83362]: 2024-04-12 08:49:44.646343929 +0000 UTC m=+0.376426177 container init a5b220c30c3fc6c5b363ae3a6fe152b1c03fcea08c515cf80ec1a40feb3f5eae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.openshift.expose-services=, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, maintainer=Paul Cuzner , name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897) 2024-04-12T08:49:44.948 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:44 smithi114 podman[83362]: 2024-04-12 08:49:44.653465654 +0000 UTC m=+0.383547901 container start a5b220c30c3fc6c5b363ae3a6fe152b1c03fcea08c515cf80ec1a40feb3f5eae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, release=239.1651231664, io.openshift.expose-services=, io.openshift.tags=base rhel8, distribution-scope=public, architecture=x86_64, description=Ceph Grafana Container, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container) 2024-04-12T08:49:44.948 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:44 smithi114 bash[83362]: a5b220c30c3fc6c5b363ae3a6fe152b1c03fcea08c515cf80ec1a40feb3f5eae 2024-04-12T08:49:44.948 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:44 smithi114 systemd[1]: Started Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:45.243 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:44 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[83375]: logger=settings t=2024-04-12T08:49:44.947822941Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-12T08:49:45.243 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:45 smithi114 podman[83391]: 2024-04-12 08:49:45.00201759 +0000 UTC m=+0.037552272 container died a5b220c30c3fc6c5b363ae3a6fe152b1c03fcea08c515cf80ec1a40feb3f5eae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.component=ubi8-container, release=239.1651231664, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., distribution-scope=public, io.buildah.version=1.32.0, io.openshift.expose-services=, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5) 2024-04-12T08:49:45.261 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:44 smithi090 podman[95797]: 2024-04-12 08:49:44.87788907 +0000 UTC m=+0.033006408 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:49:45.261 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:44 smithi090 podman[95797]: 2024-04-12 08:49:44.998821009 +0000 UTC m=+0.153938330 container create ad8458aaf57951c945065cab63904c3774082128802f354dbdd0631c3d74d44c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS) 2024-04-12T08:49:45.261 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 podman[95797]: 2024-04-12 08:49:45.260774939 +0000 UTC m=+0.415892263 container init ad8458aaf57951c945065cab63904c3774082128802f354dbdd0631c3d74d44c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-b277feb) 2024-04-12T08:49:45.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 29 16 59208 59192 127.557 82.0625 0.00709642 0.00783475 2024-04-12T08:49:45.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 podman[95797]: 2024-04-12 08:49:45.264088198 +0000 UTC m=+0.419205529 container start ad8458aaf57951c945065cab63904c3774082128802f354dbdd0631c3d74d44c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:49:45.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T08:49:45.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev), process ceph-mon, pid 2 2024-04-12T08:49:45.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: pidfile_write: ignore empty --pid-file 2024-04-12T08:49:45.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: load: jerasure load: lrc 2024-04-12T08:49:45.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: RocksDB version: 7.9.2 2024-04-12T08:49:45.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Git sha 0 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Compile date 2024-04-09 21:18:54 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: DB SUMMARY 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: DB Session ID: A8NE56DGAZI8OZOWK2C3 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: CURRENT file: CURRENT 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1179 Bytes 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 2645862 ; 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.error_if_exists: 0 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.create_if_missing: 0 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.paranoid_checks: 1 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T08:49:45.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.env: 0x5558adc7bc60 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.info_log: 0x5558af30dae0 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.statistics: (nil) 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.use_fsync: 0 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_log_file_size: 0 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.allow_fallocate: 1 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.use_direct_reads: 0 2024-04-12T08:49:45.679 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.db_log_dir: 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.wal_dir: 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.write_buffer_manager: 0x5558af2e0820 2024-04-12T08:49:45.680 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.unordered_write: 0 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.row_cache: None 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.wal_filter: None 2024-04-12T08:49:45.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.two_write_queues: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.wal_compression: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.atomic_flush: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.log_readahead_size: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T08:49:45.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_background_jobs: 2 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_background_compactions: -1 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_subcompactions: 1 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T08:49:45.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_open_files: -1 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_background_flushes: -1 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Compression algorithms supported: 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: kZSTD supported: 0 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: kXpressCompression supported: 0 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: kZlibCompression supported: 1 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: kSnappyCompression supported: 1 2024-04-12T08:49:45.684 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: kLZ4Compression supported: 1 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: kBZip2Compression supported: 0 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.merge_operator: 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_filter: None 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5558af30dbe0) 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout: cache_index_and_filter_blocks: 1 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T08:49:45.685 INFO:journalctl@ceph.mon.a.smithi090.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: pin_top_level_index_and_filter: 1 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: index_type: 0 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: data_block_index_type: 0 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: index_shortening: 1 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: checksum: 4 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: no_block_cache: 0 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache: 0x5558af311090 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache_name: BinnedLRUCache 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache_options: 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: capacity : 536870912 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: num_shard_bits : 4 2024-04-12T08:49:45.686 INFO:journalctl@ceph.mon.a.smithi090.stdout: strict_capacity_limit : 0 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: high_pri_pool_ratio: 0.000 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache_compressed: (nil) 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: persistent_cache: (nil) 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_size: 4096 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_size_deviation: 10 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_restart_interval: 16 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: index_block_restart_interval: 1 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: metadata_block_size: 4096 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: partition_filters: 0 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: use_delta_encoding: 1 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: filter_policy: bloomfilter 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: whole_key_filtering: 1 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: verify_compression: 0 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: read_amp_bytes_per_bit: 0 2024-04-12T08:49:45.687 INFO:journalctl@ceph.mon.a.smithi090.stdout: format_version: 5 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout: enable_index_compression: 1 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_align: 0 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout: max_auto_readahead_size: 262144 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout: prepopulate_block_cache: 0 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout: initial_auto_readahead_size: 8192 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compression: NoCompression 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.num_levels: 7 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T08:49:45.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T08:49:45.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T08:49:45.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T08:49:45.691 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.table_properties_collectors: 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.inplace_update_support: 0 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T08:49:45.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.bloom_locality: 0 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.max_successive_merges: 0 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.ttl: 2592000 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.enable_blob_files: false 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.min_blob_size: 0 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T08:49:45.693 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T08:49:45.694 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T08:49:45.694 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T08:49:45.694 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T08:49:45.694 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12033, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 0ab2179f-5b2c-421a-bf41-73f53e994c06 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911785306129, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911785313785, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 2449352, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12034, "largest_seqno": 13250, "table_properties": {"data_size": 2442975, "index_size": 3950, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1413, "raw_key_size": 12662, "raw_average_key_size": 22, "raw_value_size": 2430901, "raw_average_value_size": 4395, "num_data_blocks": 179, "num_entries": 553, "num_filter_entries": 553, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712911785, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "0ab2179f-5b2c-421a-bf41-73f53e994c06", "db_session_id": "A8NE56DGAZI8OZOWK2C3", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911785313916, "job": 1, "event": "recovery_finished"} 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5558af402000 2024-04-12T08:49:45.695 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: DB pointer 0x5558af3ec000 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** DB Stats ** 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T08:49:45.696 INFO:journalctl@ceph.mon.a.smithi090.stdout: L0 1/0 2.34 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 598.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: L6 1/0 6.05 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: Sum 2/0 8.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 598.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 598.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 598.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-12T08:49:45.697 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative compaction: 0.00 GB write, 118.03 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval compaction: 0.00 GB write, 118.03 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout: Block cache BinnedLRUCache@0x5558af311090#2 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: starting mon.a rank 0 at public addrs [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] at bind addrs [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: mon.a@-1(???) e3 preinit fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: mon.a@-1(???).mds e8 new map 2024-04-12T08:49:45.698 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: mon.a@-1(???).mds e8 print_map 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: e8 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: legacy client fscid: 1 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: Filesystem 'foo' (1) 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: fs_name foo 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: epoch 7 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: created 2024-04-12T08:43:28.135965+0000 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: modified 2024-04-12T08:43:37.934695+0000 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: tableserver 0 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: root 0 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: session_timeout 60 2024-04-12T08:49:45.699 INFO:journalctl@ceph.mon.a.smithi090.stdout: session_autoclose 300 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: max_file_size 1099511627776 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: max_xattr_size 65536 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: required_client_features {} 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: last_failure 0 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: last_failure_osd_epoch 0 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: max_mds 1 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: in 0 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: up {0=14631} 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: failed 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: damaged 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: stopped 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: data_pools [4] 2024-04-12T08:49:45.700 INFO:journalctl@ceph.mon.a.smithi090.stdout: metadata_pool 3 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: inline_data disabled 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: balancer 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: bal_rank_mask -1 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: standby_count_wanted 1 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: qdb_cluster leader: 0 members: 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: [mds.foo.smithi090.rxdrqj{0:14631} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: Standby daemons: 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout: [mds.foo.smithi114.nbzjob{-1:24496} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.114:6832/562045409,v1:172.21.15.114:6833/562045409] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: mon.a@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: mon.a@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: mon.a@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T08:49:45.701 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: mon.a@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T08:49:45.702 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 ceph-mon[95814]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-04-12T08:49:45.702 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 bash[95797]: ad8458aaf57951c945065cab63904c3774082128802f354dbdd0631c3d74d44c 2024-04-12T08:49:45.702 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:45 smithi090 systemd[1]: Started Ceph mon.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:46.630 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 30 16 60831 60815 126.687 101.438 0.0076772 0.00788897 2024-04-12T08:49:46.858 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:46 smithi114 podman[83391]: 2024-04-12 08:49:46.520841418 +0000 UTC m=+1.556376088 container remove a5b220c30c3fc6c5b363ae3a6fe152b1c03fcea08c515cf80ec1a40feb3f5eae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, release=239.1651231664, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.buildah.version=1.32.0, version=8.5, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, architecture=x86_64, maintainer=Paul Cuzner ) 2024-04-12T08:49:46.858 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:46 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-12T08:49:47.367 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:46 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:49:47.631 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 31 16 62086 62070 125.13 78.4375 0.00383446 0.00798609 2024-04-12T08:49:48.631 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 32 16 63454 63438 123.891 85.5 0.0232794 0.00806619 2024-04-12T08:49:48.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ignoring --setuser ceph since I am not root 2024-04-12T08:49:48.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ignoring --setgroup ceph since I am not root 2024-04-12T08:49:48.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:49:48.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:49:48.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: pgmap v38: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/2201230382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: pgmap v39: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3575302760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: pgmap v40: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3101308396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: pgmap v41: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: mon.a calling monitor election 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: monmap epoch 3 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: last_changed 2024-04-12T08:38:07.849098+0000 2024-04-12T08:49:48.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: created 2024-04-12T08:36:31.874980+0000 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: min_mon_release 17 (quincy) 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: election_strategy: 1 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: 0: [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon.a 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: 1: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.b 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: 2: [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon.c 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: mgrmap e32: y(active, since 61s), standbys: x 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: daemon grafana.a on smithi114 is in unknown state 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: mgrmap e33: y(active, since 61s), standbys: x 2024-04-12T08:49:48.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[40003]: from='mgr.25088 ' entity='' 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: pgmap v38: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2201230382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: pgmap v39: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3575302760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: pgmap v40: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3101308396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:48.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: pgmap v41: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: mon.a calling monitor election 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: monmap epoch 3 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: last_changed 2024-04-12T08:38:07.849098+0000 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: created 2024-04-12T08:36:31.874980+0000 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: min_mon_release 17 (quincy) 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: election_strategy: 1 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: 0: [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon.a 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: 1: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.b 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: 2: [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon.c 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: mgrmap e32: y(active, since 61s), standbys: x 2024-04-12T08:49:48.677 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:49:48.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T08:49:48.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: daemon grafana.a on smithi114 is in unknown state 2024-04-12T08:49:48.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: mgrmap e33: y(active, since 61s), standbys: x 2024-04-12T08:49:48.678 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:48 smithi090 ceph-mon[95814]: from='mgr.25088 ' entity='' 2024-04-12T08:49:48.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:49:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:49:48.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: ignoring --setuser ceph since I am not root 2024-04-12T08:49:48.868 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: ignoring --setgroup ceph since I am not root 2024-04-12T08:49:48.868 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mgr[79721]: -- 172.21.15.114:0/1691745310 <== mon.1 v2:172.21.15.114:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558a6a50e5a0 con 0x558a6a4e2c00 2024-04-12T08:49:48.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:49:48.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:49:48.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:49:48.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: pgmap v38: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T08:49:48.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/2201230382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:48.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: pgmap v39: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2024-04-12T08:49:48.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3575302760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:48.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: pgmap v40: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: from='mgr.25088 172.21.15.90:0/2057613266' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3101308396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: pgmap v41: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: mon.a calling monitor election 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: monmap epoch 3 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: last_changed 2024-04-12T08:38:07.849098+0000 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: created 2024-04-12T08:36:31.874980+0000 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: min_mon_release 17 (quincy) 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: election_strategy: 1 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: 0: [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon.a 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: 1: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.b 2024-04-12T08:49:48.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: 2: [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon.c 2024-04-12T08:49:48.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:49:48.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T08:49:48.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: mgrmap e32: y(active, since 61s), standbys: x 2024-04-12T08:49:48.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:49:48.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T08:49:48.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: daemon grafana.a on smithi114 is in unknown state 2024-04-12T08:49:48.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: mgrmap e33: y(active, since 61s), standbys: x 2024-04-12T08:49:48.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:48 smithi114 ceph-mon[33573]: from='mgr.25088 ' entity='' 2024-04-12T08:49:49.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:49 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:49.249+0000 7f2eaf1aa200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:49:49.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:49 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:49.393+0000 7f2eaf1aa200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:49:49.631 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 33 16 64796 64780 122.678 83.875 0.0226474 0.00814544 2024-04-12T08:49:49.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:49 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:49.407+0000 7efe6d8e3200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T08:49:49.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:49 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:49.582+0000 7efe6d8e3200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T08:49:50.175 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T08:49:50.529 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:50 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/1630214738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:50.529 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:50 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1630214738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:50.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:50 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/1630214738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:49:50.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:50 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:50.361+0000 7f2eaf1aa200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:49:50.618 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:50 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:50.507+0000 7f2eaf1aa200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:49:50.631 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 34 16 66065 66049 121.403 79.3125 0.0108513 0.00823174 2024-04-12T08:49:50.900 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:50 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:50.736+0000 7efe6d8e3200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T08:49:50.985 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch ps' 2024-04-12T08:49:51.006 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:50 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:50.721+0000 7f2eaf1aa200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:49:51.006 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:50 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:50.872+0000 7f2eaf1aa200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:49:51.006 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:51.005+0000 7f2eaf1aa200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:49:51.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:50 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:50.899+0000 7efe6d8e3200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T08:49:51.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:51 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:51.133+0000 7efe6d8e3200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T08:49:51.525 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:49:51.631 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 35 16 67290 67274 120.119 76.5625 0.00692289 0.0083196 2024-04-12T08:49:51.669 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:51.379+0000 7f2eaf1aa200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:49:51.669 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:51.535+0000 7f2eaf1aa200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:49:51.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:51 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:51.302+0000 7efe6d8e3200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T08:49:51.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:51 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:51.455+0000 7efe6d8e3200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T08:49:52.011 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:51 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:51.927+0000 7efe6d8e3200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T08:49:52.062 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:51.668+0000 7f2eaf1aa200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:49:52.062 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:51.811+0000 7f2eaf1aa200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:49:52.062 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:52 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:52.061+0000 7f2eaf1aa200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:49:52.294 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:52 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:52.130+0000 7efe6d8e3200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T08:49:52.294 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:52 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:52.293+0000 7efe6d8e3200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T08:49:52.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 36 16 68577 68561 119.017 80.4375 0.00562555 0.0083968 2024-04-12T08:49:52.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:52 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:52.469+0000 7efe6d8e3200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T08:49:53.127 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:52 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:52.749+0000 7efe6d8e3200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T08:49:53.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 37 16 69786 69770 117.842 75.5625 0.0100726 0.00848072 2024-04-12T08:49:53.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:53 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:53.508+0000 7f2eaf1aa200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:49:53.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:53 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:53.716+0000 7f2eaf1aa200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:49:54.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:54 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:54.176+0000 7f2eaf1aa200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:49:54.617 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:54 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:54.317+0000 7f2eaf1aa200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:49:54.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 38 16 71006 70990 116.748 76.25 0.00943336 0.00856069 2024-04-12T08:49:54.905 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:54 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:54.619+0000 7f2eaf1aa200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:49:54.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:54 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:54.447+0000 7efe6d8e3200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T08:49:54.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:54 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:54.668+0000 7efe6d8e3200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T08:49:55.347 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:54 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:54.903+0000 7f2eaf1aa200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:49:55.608 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:55 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:55.346+0000 7f2eaf1aa200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:49:55.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 39 16 72277 72261 115.791 79.4375 0.0135234 0.00863162 2024-04-12T08:49:55.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:55 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:55.220+0000 7efe6d8e3200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T08:49:55.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:55 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:55.378+0000 7efe6d8e3200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T08:49:55.867 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:55 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:55.718+0000 7f2eaf1aa200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:49:56.042 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:55 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:55.723+0000 7efe6d8e3200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:49:56.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:56 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:56.085+0000 7f2eaf1aa200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:49:56.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:56.041+0000 7efe6d8e3200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T08:49:56.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:49:56.632460+0000 min lat: 0.00128221 max lat: 0.20158 avg lat: 0.00869937 2024-04-12T08:49:56.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:49:56.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 40 16 73536 73520 114.863 78.6875 0.0134885 0.00869937 2024-04-12T08:49:56.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:56.556+0000 7efe6d8e3200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T08:49:57.372 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:56.954+0000 7efe6d8e3200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T08:49:57.415 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:57 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:57.358+0000 7f2eaf1aa200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:49:57.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:57 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-04-12T08:49:57.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:57 smithi114 systemd[1]: Stopped Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:57.416 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:57 smithi114 systemd[1]: Starting Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:49:57.622 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:57.371+0000 7efe6d8e3200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T08:49:57.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 41 16 74771 74755 113.944 77.1875 0.00552727 0.00877129 2024-04-12T08:49:57.783 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:57 smithi114 podman[83563]: 2024-04-12 08:49:57.31965266 +0000 UTC m=+0.025829804 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T08:49:57.783 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:57 smithi114 podman[83563]: 2024-04-12 08:49:57.479456666 +0000 UTC m=+0.185633852 container create 6aebf3532143954a6efdefd4a8734635b529ca4673f2d772efe0234ab80d25a6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, release=239.1651231664, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.openshift.tags=base rhel8, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, maintainer=Paul Cuzner , name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-12T08:49:57.783 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:57 smithi114 podman[83563]: 2024-04-12 08:49:57.782994607 +0000 UTC m=+0.489171754 container init 6aebf3532143954a6efdefd4a8734635b529ca4673f2d772efe0234ab80d25a6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, release=239.1651231664, version=8.5, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.openshift.expose-services=) 2024-04-12T08:49:58.065 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:57 smithi114 podman[83563]: 2024-04-12 08:49:57.78908102 +0000 UTC m=+0.495258158 container start 6aebf3532143954a6efdefd4a8734635b529ca4673f2d772efe0234ab80d25a6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, name=ubi8, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, distribution-scope=public, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container) 2024-04-12T08:49:58.066 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:57 smithi114 bash[83563]: 6aebf3532143954a6efdefd4a8734635b529ca4673f2d772efe0234ab80d25a6 2024-04-12T08:49:58.066 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:57 smithi114 systemd[1]: Started Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:49:58.367 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:58.239+0000 7f2eaf1aa200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:49:58.367 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[83576]: logger=settings t=2024-04-12T08:49:58.06402403Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-12T08:49:58.368 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:58 smithi114 podman[83594]: 2024-04-12 08:49:58.116525265 +0000 UTC m=+0.038439928 container died 6aebf3532143954a6efdefd4a8734635b529ca4673f2d772efe0234ab80d25a6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., distribution-scope=public, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , release=239.1651231664, name=ubi8, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git) 2024-04-12T08:49:58.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 42 16 75955 75939 112.993 74 0.013775 0.00884542 2024-04-12T08:49:58.647 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:58.396+0000 7f2eaf1aa200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:49:58.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:58.808+0000 7efe6d8e3200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T08:49:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[95814]: Standby manager daemon x restarted 2024-04-12T08:49:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[95814]: Standby manager daemon x started 2024-04-12T08:49:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[95814]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:49:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[95814]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:49:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[95814]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:49:58.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[95814]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:49:58.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[40003]: Standby manager daemon x restarted 2024-04-12T08:49:58.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[40003]: Standby manager daemon x started 2024-04-12T08:49:58.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:49:58.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:49:58.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:49:58.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:58 smithi090 ceph-mon[40003]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:49:58.947 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-mon[33573]: Standby manager daemon x restarted 2024-04-12T08:49:58.947 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-mon[33573]: Standby manager daemon x started 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-mon[33573]: from='mgr.? 172.21.15.114:0/1552260936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:58.726+0000 7f2eaf1aa200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: [12/Apr/2024:08:49:58] ENGINE Bus STARTING 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: CherryPy Checker: 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: The Application mounted at '' has an empty config. 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: [12/Apr/2024:08:49:58] ENGINE Serving on http://:::9283 2024-04-12T08:49:58.948 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 08:49:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x[79717]: [12/Apr/2024:08:49:58] ENGINE Bus STARTED 2024-04-12T08:49:59.288 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:58 smithi114 podman[83594]: 2024-04-12 08:49:58.947508814 +0000 UTC m=+0.869423470 container remove 6aebf3532143954a6efdefd4a8734635b529ca4673f2d772efe0234ab80d25a6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, vcs-type=git, build-date=2022-04-29T11:48:11.055897, name=ubi8, maintainer=Paul Cuzner , description=Ceph Grafana Container, distribution-scope=public, io.buildah.version=1.32.0, release=239.1651231664, version=8.5, vendor=Red Hat, Inc., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8) 2024-04-12T08:49:59.288 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:58 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-12T08:49:59.617 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:49:59 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:49:59.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 43 16 77160 77144 112.116 75.3125 0.00605087 0.00891452 2024-04-12T08:50:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:49:59 smithi114 ceph-mon[33573]: mgrmap e34: y(active, since 72s), standbys: x 2024-04-12T08:50:00.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:49:59 smithi090 ceph-mon[95814]: mgrmap e34: y(active, since 72s), standbys: x 2024-04-12T08:50:00.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:59 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:59.744+0000 7efe6d8e3200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T08:50:00.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:49:59 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:49:59.913+0000 7efe6d8e3200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T08:50:00.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:49:59 smithi090 ceph-mon[40003]: mgrmap e34: y(active, since 72s), standbys: x 2024-04-12T08:50:00.632 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 44 16 78308 78292 111.199 71.75 0.0212149 0.00898705 2024-04-12T08:50:00.650 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.302+0000 7efe6d8e3200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T08:50:00.650 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:50:00] ENGINE Bus STARTING 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.745+0000 7efdd5bcf700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.745+0000 7efdd5bcf700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.745+0000 7efdd5bcf700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.745+0000 7efdd5bcf700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.745+0000 7efdd5bcf700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.747+0000 7efdcabb9700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.747+0000 7efdcabb9700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.747+0000 7efdcabb9700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.747+0000 7efdcabb9700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.747+0000 7efdcabb9700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: CherryPy Checker: 2024-04-12T08:50:00.925 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: The Application mounted at '' has an empty config. 2024-04-12T08:50:00.926 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:00.926 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:50:00] ENGINE Serving on http://:::9283 2024-04-12T08:50:00.926 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:50:00] ENGINE Bus STARTED 2024-04-12T08:50:00.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:50:00.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: Active manager daemon y restarted 2024-04-12T08:50:00.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: Activating manager daemon y 2024-04-12T08:50:00.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T08:50:00.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: mgrmap e35: y(active, starting, since 0.0201704s), standbys: x 2024-04-12T08:50:00.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:50:00.927 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: Manager daemon y is now available 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: Active manager daemon y restarted 2024-04-12T08:50:00.928 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: Activating manager daemon y 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: mgrmap e35: y(active, starting, since 0.0201704s), standbys: x 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:50:00.929 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:50:00.930 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:50:00.930 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:50:00.930 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:50:00.930 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: Manager daemon y is now available 2024-04-12T08:50:00.930 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:00.930 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:50:00.930 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:50:00.930 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:50:00.930 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:50:00.930 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:00 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:50:00.954 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: Active manager daemon y restarted 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: Activating manager daemon y 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: mgrmap e35: y(active, starting, since 0.0201704s), standbys: x 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi090.rxdrqj"}]: dispatch 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi114.nbzjob"}]: dispatch 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T08:50:00.955 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: Manager daemon y is now available 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:50:00.956 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T08:50:00.957 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:50:00.957 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:00 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T08:50:01.368 INFO:teuthology.orchestra.run.smithi090.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T08:50:01.368 INFO:teuthology.orchestra.run.smithi090.stdout:alertmanager.a smithi090 *:9093,9094 running (2m) 68s ago 7m 16.8M - 0.25.0 c8568f914cd2 d11ba2dc752a 2024-04-12T08:50:01.368 INFO:teuthology.orchestra.run.smithi090.stdout:grafana.a smithi114 *:3000 unknown 39s ago 7m - - 954c08fa6188 bcf41ef25c38 2024-04-12T08:50:01.368 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi090.rxdrqj smithi090 running (6m) 68s ago 6m 18.3M - 17.2.7-827-gd3b5060a 21de9b7369ad f627bce7c272 2024-04-12T08:50:01.368 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi114.nbzjob smithi114 running (6m) 39s ago 6m 13.7M - 17.2.7-827-gd3b5060a 21de9b7369ad 60cb65499580 2024-04-12T08:50:01.368 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.x smithi114 *:8443,9283,8765 running (46s) 39s ago 11m 66.1M - 19.0.0-2053-gb277feb2 668895944f0e 60c0ca408f91 2024-04-12T08:50:01.368 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.y smithi090 *:8443,9283,8765 running (88s) 68s ago 13m 480M - 19.0.0-2053-gb277feb2 668895944f0e b61a0f582c61 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:mon.a smithi090 running (13m) 68s ago 13m 52.3M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 5c90fd1861be 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:mon.b smithi114 running (12m) 39s ago 12m 42.5M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 63cd6193074e 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:mon.c smithi090 running (11m) 68s ago 11m 39.2M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad d5aa934587fe 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.a smithi090 *:9100 running (2m) 68s ago 7m 8220k - 1.5.0 0da6a335fe13 687747c6880c 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.b smithi114 *:9100 running (114s) 39s ago 7m 8866k - 1.5.0 0da6a335fe13 e89762981b55 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:osd.0 smithi090 running (11m) 68s ago 11m 381M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 9395ed639a37 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:osd.1 smithi090 running (10m) 68s ago 10m 256M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 98eb8b5854e9 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:osd.2 smithi090 running (10m) 68s ago 10m 320M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 8008248b4e61 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:osd.3 smithi090 running (10m) 68s ago 10m 174M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 83a5641ba19a 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:osd.4 smithi114 running (9m) 39s ago 9m 294M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 15d0d8563392 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:osd.5 smithi114 running (9m) 39s ago 9m 310M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 54049e134882 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:osd.6 smithi114 running (8m) 39s ago 8m 174M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 86dfedc8629b 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:osd.7 smithi114 running (8m) 39s ago 8m 206M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad dda597b19712 2024-04-12T08:50:01.369 INFO:teuthology.orchestra.run.smithi090.stdout:prometheus.a smithi114 *:9095 running (52s) 39s ago 8m 46.5M - 2.43.0 a07b618ecd1d f009118ca428 2024-04-12T08:50:01.633 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 45 16 79274 79258 110.069 60.375 0.00972533 0.00907981 2024-04-12T08:50:01.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:01 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:50:01.355+0000 7efe43a91700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-12T08:50:02.202 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade pause' 2024-04-12T08:50:02.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:02 smithi114 ceph-mon[33573]: from='client.34116 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:50:02.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:02 smithi114 ceph-mon[33573]: mgrmap e36: y(active, since 1.03375s), standbys: x 2024-04-12T08:50:02.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:02 smithi114 ceph-mon[33573]: pgmap v3: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-04-12T08:50:02.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:02 smithi090 ceph-mon[95814]: from='client.34116 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:50:02.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:02 smithi090 ceph-mon[95814]: mgrmap e36: y(active, since 1.03375s), standbys: x 2024-04-12T08:50:02.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:02 smithi090 ceph-mon[95814]: pgmap v3: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-04-12T08:50:02.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:02 smithi090 ceph-mon[40003]: from='client.34116 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:50:02.425 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:02 smithi090 ceph-mon[40003]: mgrmap e36: y(active, since 1.03375s), standbys: x 2024-04-12T08:50:02.425 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:02 smithi090 ceph-mon[40003]: pgmap v3: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-04-12T08:50:02.633 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 46 16 80308 80292 109.08 64.625 0.01047 0.00916163 2024-04-12T08:50:03.361 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:50:03.556 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[95814]: [12/Apr/2024:08:50:02] ENGINE Bus STARTING 2024-04-12T08:50:03.556 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[95814]: [12/Apr/2024:08:50:02] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:50:03.556 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[95814]: [12/Apr/2024:08:50:02] ENGINE Serving on http://172.21.15.90:8765 2024-04-12T08:50:03.556 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[95814]: [12/Apr/2024:08:50:02] ENGINE Bus STARTED 2024-04-12T08:50:03.556 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[95814]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T08:50:03.557 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[95814]: Cluster is now healthy 2024-04-12T08:50:03.557 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[95814]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-12T08:50:03.557 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[95814]: pgmap v4: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-04-12T08:50:03.557 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:50:03] "GET /metrics HTTP/1.1" 200 33548 "" "Prometheus/2.43.0" 2024-04-12T08:50:03.557 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[40003]: [12/Apr/2024:08:50:02] ENGINE Bus STARTING 2024-04-12T08:50:03.557 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[40003]: [12/Apr/2024:08:50:02] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:50:03.557 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[40003]: [12/Apr/2024:08:50:02] ENGINE Serving on http://172.21.15.90:8765 2024-04-12T08:50:03.557 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[40003]: [12/Apr/2024:08:50:02] ENGINE Bus STARTED 2024-04-12T08:50:03.557 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[40003]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T08:50:03.558 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[40003]: Cluster is now healthy 2024-04-12T08:50:03.558 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[40003]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-12T08:50:03.558 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:03 smithi090 ceph-mon[40003]: pgmap v4: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-04-12T08:50:03.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:03 smithi114 ceph-mon[33573]: [12/Apr/2024:08:50:02] ENGINE Bus STARTING 2024-04-12T08:50:03.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:03 smithi114 ceph-mon[33573]: [12/Apr/2024:08:50:02] ENGINE Serving on https://172.21.15.90:7150 2024-04-12T08:50:03.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:03 smithi114 ceph-mon[33573]: [12/Apr/2024:08:50:02] ENGINE Serving on http://172.21.15.90:8765 2024-04-12T08:50:03.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:03 smithi114 ceph-mon[33573]: [12/Apr/2024:08:50:02] ENGINE Bus STARTED 2024-04-12T08:50:03.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:03 smithi114 ceph-mon[33573]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T08:50:03.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:03 smithi114 ceph-mon[33573]: Cluster is now healthy 2024-04-12T08:50:03.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:03 smithi114 ceph-mon[33573]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-12T08:50:03.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:03 smithi114 ceph-mon[33573]: pgmap v4: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-04-12T08:50:03.633 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 47 15 81532 81517 108.388 76.5625 0.00807349 0.00922114 2024-04-12T08:50:04.633 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 48 16 82829 82813 107.817 81 0.00653525 0.00926983 2024-04-12T08:50:05.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:05 smithi114 ceph-mon[33573]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-12T08:50:05.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:05 smithi114 ceph-mon[33573]: pgmap v5: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-04-12T08:50:05.633 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 49 16 83928 83912 107.019 68.6875 0.00346407 0.00933502 2024-04-12T08:50:05.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:05 smithi090 ceph-mon[40003]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-12T08:50:05.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:05 smithi090 ceph-mon[40003]: pgmap v5: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-04-12T08:50:05.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:05 smithi090 ceph-mon[95814]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-12T08:50:05.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:05 smithi090 ceph-mon[95814]: pgmap v5: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-04-12T08:50:06.634 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 50 16 84776 84760 105.938 53 0.00799273 0.0094344 2024-04-12T08:50:07.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:06 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:07.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:06 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:07.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:06 smithi114 ceph-mon[33573]: pgmap v6: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 KiB/s rd, 35 MiB/s wr, 582 op/s 2024-04-12T08:50:07.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:06 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:07.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:06 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:07.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:06 smithi090 ceph-mon[40003]: pgmap v6: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 KiB/s rd, 35 MiB/s wr, 582 op/s 2024-04-12T08:50:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:06 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:06 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:06 smithi090 ceph-mon[95814]: pgmap v6: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 KiB/s rd, 35 MiB/s wr, 582 op/s 2024-04-12T08:50:07.633 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 51 16 85678 85662 104.966 56.375 0.00460816 0.00952162 2024-04-12T08:50:08.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:08 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:08.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:08 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:08.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:08 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:08.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:08 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:08.491 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:08 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:08.491 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:08 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:08.634 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 52 16 86882 86866 104.394 75.25 0.00465546 0.00957436 2024-04-12T08:50:08.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:50:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:50:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:50:09.583 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:09 smithi114 ceph-mon[33573]: pgmap v7: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 30 KiB/s rd, 49 MiB/s wr, 806 op/s 2024-04-12T08:50:09.583 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:09 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-04-12T08:50:09.583 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:09 smithi114 systemd[1]: Stopped Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:50:09.583 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:09 smithi114 systemd[1]: Starting Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:50:09.630 INFO:teuthology.orchestra.run.smithi090.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:50:09.630 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:09 smithi090 ceph-mon[95814]: pgmap v7: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 30 KiB/s rd, 49 MiB/s wr, 806 op/s 2024-04-12T08:50:09.631 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:09 smithi090 ceph-mon[40003]: pgmap v7: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 30 KiB/s rd, 49 MiB/s wr, 806 op/s 2024-04-12T08:50:09.634 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 53 16 87944 87928 103.676 66.375 0.0154663 0.00963939 2024-04-12T08:50:10.499 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:10 smithi114 podman[84525]: 2024-04-12 08:50:10.065196556 +0000 UTC m=+0.031118249 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T08:50:10.499 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:10 smithi114 podman[84525]: 2024-04-12 08:50:10.203279949 +0000 UTC m=+0.169201674 container create da1e03da6b4cc46bd332d6cd369ef88a96a146f2ccc7db9ff44dd9a0be715d12 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, version=8.5, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, name=ubi8, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, maintainer=Paul Cuzner , architecture=x86_64, io.buildah.version=1.32.0, vendor=Red Hat, Inc.) 2024-04-12T08:50:10.499 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:10 smithi114 podman[84525]: 2024-04-12 08:50:10.49856416 +0000 UTC m=+0.464485856 container init da1e03da6b4cc46bd332d6cd369ef88a96a146f2ccc7db9ff44dd9a0be715d12 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , architecture=x86_64, vcs-type=git, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, version=8.5, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container) 2024-04-12T08:50:10.634 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 54 16 88929 88913 102.896 61.5625 0.00435524 0.00968556 2024-04-12T08:50:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:10 smithi114 ceph-mon[33573]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:50:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:10 smithi114 ceph-mon[33573]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:50:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:10 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:10 smithi114 ceph-mon[33573]: pgmap v8: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 23 KiB/s rd, 38 MiB/s wr, 626 op/s 2024-04-12T08:50:10.868 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:10 smithi114 podman[84525]: 2024-04-12 08:50:10.502265324 +0000 UTC m=+0.468187010 container start da1e03da6b4cc46bd332d6cd369ef88a96a146f2ccc7db9ff44dd9a0be715d12 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.openshift.expose-services=, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, version=8.5, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, description=Ceph Grafana Container, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-12T08:50:10.868 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:10 smithi114 bash[84525]: da1e03da6b4cc46bd332d6cd369ef88a96a146f2ccc7db9ff44dd9a0be715d12 2024-04-12T08:50:10.868 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:10 smithi114 systemd[1]: Started Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:50:10.868 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:10 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a[84545]: logger=settings t=2024-04-12T08:50:10.658540058Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-12T08:50:10.868 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:10 smithi114 podman[85760]: 2024-04-12 08:50:10.712657508 +0000 UTC m=+0.044389125 container died da1e03da6b4cc46bd332d6cd369ef88a96a146f2ccc7db9ff44dd9a0be715d12 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.buildah.version=1.32.0, description=Ceph Grafana Container, io.openshift.expose-services=, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-12T08:50:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:10 smithi090 ceph-mon[95814]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:50:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:10 smithi090 ceph-mon[95814]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:50:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:10 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:10 smithi090 ceph-mon[95814]: pgmap v8: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 23 KiB/s rd, 38 MiB/s wr, 626 op/s 2024-04-12T08:50:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:10 smithi090 ceph-mon[40003]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:50:10.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:10 smithi090 ceph-mon[40003]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:50:10.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:10 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:10.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:10 smithi090 ceph-mon[40003]: pgmap v8: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 23 KiB/s rd, 38 MiB/s wr, 626 op/s 2024-04-12T08:50:11.617 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:11 smithi114 podman[85760]: 2024-04-12 08:50:11.347773167 +0000 UTC m=+0.679504782 container remove da1e03da6b4cc46bd332d6cd369ef88a96a146f2ccc7db9ff44dd9a0be715d12 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, maintainer=Paul Cuzner , distribution-scope=public, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, vcs-type=git, vendor=Red Hat, Inc., description=Ceph Grafana Container, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, io.buildah.version=1.32.0, release=239.1651231664) 2024-04-12T08:50:11.617 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:11 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-12T08:50:11.634 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 55 16 90094 90078 102.349 72.8125 0.0170476 0.00976571 2024-04-12T08:50:12.025 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'sleep 60' 2024-04-12T08:50:12.367 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:12 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:50:12.636 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 56 16 91232 91216 101.789 71.125 0.00722039 0.00981946 2024-04-12T08:50:12.962 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:50:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:13 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:13 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:13 smithi114 ceph-mon[33573]: pgmap v9: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 19 KiB/s rd, 61 MiB/s wr, 993 op/s 2024-04-12T08:50:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:13 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:13 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.385 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:50:13] "GET /metrics HTTP/1.1" 200 33548 "" "Prometheus/2.43.0" 2024-04-12T08:50:13.385 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.386 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.386 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-mon[40003]: pgmap v9: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 19 KiB/s rd, 61 MiB/s wr, 993 op/s 2024-04-12T08:50:13.386 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.386 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.386 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.386 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.387 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-mon[95814]: pgmap v9: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 19 KiB/s rd, 61 MiB/s wr, 993 op/s 2024-04-12T08:50:13.387 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.387 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:13.636 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 57 16 92287 92271 101.16 65.9375 0.00994487 0.00987966 2024-04-12T08:50:14.636 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 58 16 93404 93388 100.62 69.8125 0.00592023 0.00993427 2024-04-12T08:50:15.636 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 59 16 94434 94418 100.005 64.375 0.0125142 0.00999436 2024-04-12T08:50:15.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:15 smithi090 ceph-mon[40003]: pgmap v10: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 17 KiB/s rd, 56 MiB/s wr, 907 op/s 2024-04-12T08:50:15.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:15 smithi090 ceph-mon[95814]: pgmap v10: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 17 KiB/s rd, 56 MiB/s wr, 907 op/s 2024-04-12T08:50:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:15 smithi114 ceph-mon[33573]: pgmap v10: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 17 KiB/s rd, 56 MiB/s wr, 907 op/s 2024-04-12T08:50:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:16 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:16 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:16 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:16.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:16 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:16.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:16 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:50:16.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:16 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:50:16.636 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:50:16.636663+0000 min lat: 0.00128221 max lat: 0.234554 avg lat: 0.0100622 2024-04-12T08:50:16.636 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:50:16.636 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 60 16 95388 95372 99.332 59.625 0.0113081 0.0100622 2024-04-12T08:50:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:50:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:50:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:16.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:16.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:16.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:50:16.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:16 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:50:17.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 61 16 96418 96402 98.7571 64.375 0.00524095 0.0101209 2024-04-12T08:50:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: pgmap v11: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 17 KiB/s rd, 73 MiB/s wr, 1.18k op/s 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: pgmap v12: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-04-12T08:50:17.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:17 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:50:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:50:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:50:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:50:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:50:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: pgmap v11: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 17 KiB/s rd, 73 MiB/s wr, 1.18k op/s 2024-04-12T08:50:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:50:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: pgmap v12: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:50:17.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: pgmap v11: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 17 KiB/s rd, 73 MiB/s wr, 1.18k op/s 2024-04-12T08:50:17.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-12T08:50:17.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: Updating smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:50:17.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: Updating smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T08:50:17.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:17.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: pgmap v12: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-04-12T08:50:17.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:17 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:18.637 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 62 16 97609 97593 98.3647 74.4375 0.0175532 0.0101613 2024-04-12T08:50:18.825 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:50:18.825 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:50:18.825 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:50:18.825 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[40003]: pgmap v13: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-12T08:50:18.825 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[40003]: pgmap v14: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-12T08:50:18.825 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:18.825 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[40003]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T08:50:18.825 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:50:18.825 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:50:18.826 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:50:18.826 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[95814]: pgmap v13: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-12T08:50:18.826 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[95814]: pgmap v14: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-12T08:50:18.826 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:18.826 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:18 smithi090 ceph-mon[95814]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T08:50:18.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:50:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:50:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:50:18.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:18 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:50:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:18 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:50:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:18 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:50:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:18 smithi114 ceph-mon[33573]: pgmap v13: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-12T08:50:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:18 smithi114 ceph-mon[33573]: pgmap v14: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-12T08:50:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:18 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:18 smithi114 ceph-mon[33573]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T08:50:19.637 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 63 16 98785 98769 97.97 73.5 0.0100368 0.0102027 2024-04-12T08:50:20.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 64 16 99815 99799 97.4451 64.375 0.012696 0.0102563 2024-04-12T08:50:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:20 smithi090 ceph-mon[40003]: pgmap v15: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2024-04-12T08:50:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:20 smithi090 ceph-mon[95814]: pgmap v15: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2024-04-12T08:50:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:20 smithi114 ceph-mon[33573]: pgmap v15: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2024-04-12T08:50:21.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 65 16 100806 100790 96.8987 61.9375 0.00871395 0.0103146 2024-04-12T08:50:22.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 66 16 101905 101889 96.4712 68.6875 0.0159789 0.010361 2024-04-12T08:50:22.655 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:22 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 9. 2024-04-12T08:50:22.655 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:22 smithi114 systemd[1]: Stopped Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:50:22.655 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:22 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Start request repeated too quickly. 2024-04-12T08:50:22.655 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:22 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T08:50:22.655 INFO:journalctl@ceph.grafana.a.smithi114.stdout:Apr 12 08:50:22 smithi114 systemd[1]: Failed to start Ceph grafana.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:50:22.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:22 smithi090 ceph-mon[95814]: pgmap v16: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-12T08:50:22.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:22 smithi090 ceph-mon[40003]: pgmap v16: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-12T08:50:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:22 smithi114 ceph-mon[33573]: pgmap v16: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-12T08:50:23.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 67 16 103023 103007 96.0742 69.875 0.0193531 0.0104037 2024-04-12T08:50:23.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:50:23] "GET /metrics HTTP/1.1" 200 36270 "" "Prometheus/2.43.0" 2024-04-12T08:50:24.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 68 16 104140 104124 95.6879 69.8125 0.0176001 0.0104456 2024-04-12T08:50:24.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:24 smithi090 ceph-mon[95814]: pgmap v17: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T08:50:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:24 smithi090 ceph-mon[40003]: pgmap v17: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T08:50:25.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:24 smithi114 ceph-mon[33573]: pgmap v17: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T08:50:25.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 69 16 105280 105264 95.3337 71.25 0.0147933 0.0104844 2024-04-12T08:50:26.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 70 16 106388 106372 94.961 69.25 0.0395906 0.0105231 2024-04-12T08:50:26.917 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:26 smithi114 ceph-mon[33573]: pgmap v18: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 663 op/s 2024-04-12T08:50:26.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:26 smithi090 ceph-mon[95814]: pgmap v18: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 663 op/s 2024-04-12T08:50:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:26 smithi090 ceph-mon[40003]: pgmap v18: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 663 op/s 2024-04-12T08:50:27.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 71 16 107488 107472 94.5918 68.75 0.017807 0.0105671 2024-04-12T08:50:28.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 72 16 108568 108552 94.2154 67.5 0.0122023 0.0106086 2024-04-12T08:50:28.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:28 smithi090 ceph-mon[95814]: pgmap v19: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:50:28.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:28 smithi090 ceph-mon[40003]: pgmap v19: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:50:29.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:50:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:50:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:50:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:28 smithi114 ceph-mon[33573]: pgmap v19: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:50:29.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 73 16 109650 109634 93.8511 67.625 0.00828784 0.0106497 2024-04-12T08:50:30.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 74 16 110718 110702 93.4848 66.75 0.0136037 0.0106923 2024-04-12T08:50:31.638 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 75 16 111767 111751 93.1125 65.5625 0.0158854 0.0107351 2024-04-12T08:50:31.858 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:31 smithi090 ceph-mon[40003]: pgmap v20: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-04-12T08:50:31.858 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:31 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:31.858 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:31 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:31.858 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:31 smithi090 ceph-mon[95814]: pgmap v20: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-04-12T08:50:31.859 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:31 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:31.859 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:31 smithi114 ceph-mon[33573]: pgmap v20: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-04-12T08:50:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:31 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:50:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:31 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:32.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 76 16 112733 112717 92.6817 60.375 0.0104514 0.0107848 2024-04-12T08:50:33.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 77 16 113777 113761 92.3254 65.25 0.0077397 0.0108264 2024-04-12T08:50:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:50:33] "GET /metrics HTTP/1.1" 200 36271 "" "Prometheus/2.43.0" 2024-04-12T08:50:33.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:33 smithi090 ceph-mon[95814]: pgmap v21: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-12T08:50:33.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:33 smithi090 ceph-mon[40003]: pgmap v21: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-12T08:50:33.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:33 smithi114 ceph-mon[33573]: pgmap v21: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-12T08:50:34.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 78 16 114814 114798 91.9726 64.8125 0.0235066 0.0108682 2024-04-12T08:50:35.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 79 16 115825 115809 91.6082 63.1875 0.00622326 0.0109106 2024-04-12T08:50:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:35 smithi114 ceph-mon[33573]: pgmap v22: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-04-12T08:50:35.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:35 smithi090 ceph-mon[40003]: pgmap v22: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-04-12T08:50:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:35 smithi090 ceph-mon[95814]: pgmap v22: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-04-12T08:50:36.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:50:36.639525+0000 min lat: 0.00128221 max lat: 0.234554 avg lat: 0.010952 2024-04-12T08:50:36.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:50:36.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 80 16 116848 116832 91.2622 63.9375 0.00679021 0.010952 2024-04-12T08:50:37.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 81 16 117871 117855 90.9248 63.9375 0.0105114 0.0109929 2024-04-12T08:50:37.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:37 smithi114 ceph-mon[33573]: pgmap v23: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-04-12T08:50:37.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:37 smithi090 ceph-mon[40003]: pgmap v23: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-04-12T08:50:37.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:37 smithi090 ceph-mon[95814]: pgmap v23: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-04-12T08:50:38.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 82 16 118828 118812 90.5453 59.8125 0.0123283 0.0110391 2024-04-12T08:50:39.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:50:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:50:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:50:39.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 83 16 119812 119796 90.1954 61.5 0.021533 0.0110823 2024-04-12T08:50:39.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:39 smithi114 ceph-mon[33573]: pgmap v24: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:50:39.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:39 smithi090 ceph-mon[40003]: pgmap v24: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:50:39.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:39 smithi090 ceph-mon[95814]: pgmap v24: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:50:40.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 84 16 120807 120791 89.8619 62.1875 0.0222051 0.011123 2024-04-12T08:50:41.640 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 85 16 121808 121792 89.5407 62.5625 0.0083923 0.0111623 2024-04-12T08:50:41.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:41 smithi114 ceph-mon[33573]: pgmap v25: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-04-12T08:50:41.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:41 smithi090 ceph-mon[40003]: pgmap v25: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-04-12T08:50:41.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:41 smithi090 ceph-mon[95814]: pgmap v25: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-04-12T08:50:42.639 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 86 16 122803 122787 89.2226 62.1875 0.00939176 0.0112028 2024-04-12T08:50:43.640 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 87 16 123760 123744 88.8845 59.8125 0.0122668 0.0112453 2024-04-12T08:50:43.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:50:43] "GET /metrics HTTP/1.1" 200 36271 "" "Prometheus/2.43.0" 2024-04-12T08:50:43.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:43 smithi090 ceph-mon[40003]: pgmap v26: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-04-12T08:50:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:43 smithi090 ceph-mon[95814]: pgmap v26: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-04-12T08:50:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:43 smithi114 ceph-mon[33573]: pgmap v26: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-04-12T08:50:44.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 88 16 124739 124723 88.5697 61.1875 0.00934057 0.0112857 2024-04-12T08:50:45.640 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 89 16 125717 125701 88.2613 61.125 0.0142055 0.0113254 2024-04-12T08:50:45.671 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:45 smithi114 ceph-mon[33573]: pgmap v27: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-12T08:50:45.671 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:45 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:45 smithi090 ceph-mon[40003]: pgmap v27: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-12T08:50:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:45 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:45 smithi090 ceph-mon[95814]: pgmap v27: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-12T08:50:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:50:46.640 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Total time run: 90.0075 2024-04-12T08:50:46.640 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Total writes made: 126675 2024-04-12T08:50:46.640 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Write size: 65536 2024-04-12T08:50:46.640 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Object size: 65536 2024-04-12T08:50:46.640 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Bandwidth (MB/sec): 87.9614 2024-04-12T08:50:46.640 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev Bandwidth: 38.1942 2024-04-12T08:50:46.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max bandwidth (MB/sec): 228.25 2024-04-12T08:50:46.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min bandwidth (MB/sec): 53 2024-04-12T08:50:46.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Average IOPS: 1407 2024-04-12T08:50:46.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev IOPS: 611.107 2024-04-12T08:50:46.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max IOPS: 3652 2024-04-12T08:50:46.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min IOPS: 848 2024-04-12T08:50:46.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Average Latency(s): 0.0113644 2024-04-12T08:50:46.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev Latency(s): 0.0100841 2024-04-12T08:50:46.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max latency(s): 0.234554 2024-04-12T08:50:46.641 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min latency(s): 0.00128221 2024-04-12T08:50:46.644 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Cleaning up (deleting benchmark objects) 2024-04-12T08:50:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:47 smithi114 ceph-mon[33573]: pgmap v28: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-12T08:50:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:47 smithi090 ceph-mon[40003]: pgmap v28: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-12T08:50:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:47 smithi090 ceph-mon[95814]: pgmap v28: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-12T08:50:48.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:50:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:50:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:50:49.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:49 smithi114 ceph-mon[33573]: pgmap v29: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-12T08:50:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:49 smithi090 ceph-mon[40003]: pgmap v29: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-12T08:50:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:49 smithi090 ceph-mon[95814]: pgmap v29: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-12T08:50:51.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:51 smithi114 ceph-mon[33573]: pgmap v30: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-04-12T08:50:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:51 smithi090 ceph-mon[40003]: pgmap v30: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-04-12T08:50:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:51 smithi090 ceph-mon[95814]: pgmap v30: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-04-12T08:50:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:50:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:50:53] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.43.0" 2024-04-12T08:50:53.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:53 smithi090 ceph-mon[40003]: pgmap v31: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 53 MiB/s wr, 1.24k op/s 2024-04-12T08:50:53.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:53 smithi090 ceph-mon[95814]: pgmap v31: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 53 MiB/s wr, 1.24k op/s 2024-04-12T08:50:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:53 smithi114 ceph-mon[33573]: pgmap v31: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 53 MiB/s wr, 1.24k op/s 2024-04-12T08:50:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:54 smithi114 ceph-mon[33573]: pgmap v32: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 821 op/s 2024-04-12T08:50:54.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:54 smithi090 ceph-mon[40003]: pgmap v32: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 821 op/s 2024-04-12T08:50:54.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:54 smithi090 ceph-mon[95814]: pgmap v32: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 821 op/s 2024-04-12T08:50:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:56 smithi090 ceph-mon[40003]: pgmap v33: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 821 op/s 2024-04-12T08:50:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:56 smithi090 ceph-mon[95814]: pgmap v33: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 821 op/s 2024-04-12T08:50:57.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:56 smithi114 ceph-mon[33573]: pgmap v33: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 821 op/s 2024-04-12T08:50:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:50:58 smithi114 ceph-mon[33573]: pgmap v34: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 1.23k op/s 2024-04-12T08:50:58.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:50:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:50:58.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:50:58.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:50:58 smithi090 ceph-mon[40003]: pgmap v34: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 1.23k op/s 2024-04-12T08:50:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:50:58 smithi090 ceph-mon[95814]: pgmap v34: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 1.23k op/s 2024-04-12T08:51:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:00 smithi090 ceph-mon[40003]: pgmap v35: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.2 MiB/s wr, 820 op/s 2024-04-12T08:51:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:00 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:00 smithi090 ceph-mon[95814]: pgmap v35: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.2 MiB/s wr, 820 op/s 2024-04-12T08:51:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:00 smithi114 ceph-mon[33573]: pgmap v35: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.2 MiB/s wr, 820 op/s 2024-04-12T08:51:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:02 smithi114 ceph-mon[33573]: pgmap v36: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 1.2 MiB/s wr, 1.22k op/s 2024-04-12T08:51:03.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:02 smithi090 ceph-mon[95814]: pgmap v36: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 1.2 MiB/s wr, 1.22k op/s 2024-04-12T08:51:03.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:02 smithi090 ceph-mon[40003]: pgmap v36: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 1.2 MiB/s wr, 1.22k op/s 2024-04-12T08:51:03.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:51:03] "GET /metrics HTTP/1.1" 200 36270 "" "Prometheus/2.43.0" 2024-04-12T08:51:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:04 smithi114 ceph-mon[33573]: pgmap v37: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-04-12T08:51:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:04 smithi090 ceph-mon[95814]: pgmap v37: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-04-12T08:51:05.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:04 smithi090 ceph-mon[40003]: pgmap v37: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-04-12T08:51:07.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:06 smithi114 ceph-mon[33573]: pgmap v38: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-04-12T08:51:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:06 smithi090 ceph-mon[95814]: pgmap v38: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-04-12T08:51:07.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:06 smithi090 ceph-mon[40003]: pgmap v38: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-04-12T08:51:09.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:08 smithi114 ceph-mon[33573]: pgmap v39: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-04-12T08:51:09.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:51:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:51:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:51:09.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:08 smithi090 ceph-mon[40003]: pgmap v39: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-04-12T08:51:09.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:08 smithi090 ceph-mon[95814]: pgmap v39: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-04-12T08:51:11.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:10 smithi114 ceph-mon[33573]: pgmap v40: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2024-04-12T08:51:11.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:10 smithi090 ceph-mon[40003]: pgmap v40: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2024-04-12T08:51:11.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:10 smithi090 ceph-mon[95814]: pgmap v40: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2024-04-12T08:51:13.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:12 smithi114 ceph-mon[33573]: pgmap v41: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-04-12T08:51:13.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:12 smithi090 ceph-mon[40003]: pgmap v41: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-04-12T08:51:13.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:12 smithi090 ceph-mon[95814]: pgmap v41: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-04-12T08:51:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:51:13] "GET /metrics HTTP/1.1" 200 36270 "" "Prometheus/2.43.0" 2024-04-12T08:51:15.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:14 smithi114 ceph-mon[33573]: pgmap v42: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-12T08:51:15.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:14 smithi090 ceph-mon[40003]: pgmap v42: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-12T08:51:15.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:14 smithi090 ceph-mon[95814]: pgmap v42: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-12T08:51:16.018 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:15 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:16.134 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:15 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:16.135 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:16.176 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade resume' 2024-04-12T08:51:16.624 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:51:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:16 smithi114 ceph-mon[33573]: pgmap v43: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-12T08:51:17.118 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:16 smithi090 ceph-mon[40003]: pgmap v43: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-12T08:51:17.119 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:16 smithi090 ceph-mon[95814]: pgmap v43: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-12T08:51:18.933 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:51:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:51:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:51:18.933 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:18 smithi114 ceph-mon[33573]: pgmap v44: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-04-12T08:51:18.933 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:18 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:51:19.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:18 smithi090 ceph-mon[40003]: pgmap v44: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-04-12T08:51:19.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:18 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:51:19.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:18 smithi090 ceph-mon[95814]: pgmap v44: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-04-12T08:51:19.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:18 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:51:20.791 INFO:teuthology.orchestra.run.smithi090.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:51:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:20 smithi114 ceph-mon[33573]: pgmap v45: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2024-04-12T08:51:21.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:20 smithi090 ceph-mon[40003]: pgmap v45: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2024-04-12T08:51:21.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:20 smithi090 ceph-mon[95814]: pgmap v45: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2024-04-12T08:51:22.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:21 smithi114 ceph-mon[33573]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:51:22.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:21 smithi114 ceph-mon[33573]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:51:22.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:21 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:22.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:21 smithi090 ceph-mon[40003]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:51:22.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:21 smithi090 ceph-mon[40003]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:51:22.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:21 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:22.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:21 smithi090 ceph-mon[95814]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:51:22.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:21 smithi090 ceph-mon[95814]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:51:22.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:21 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:23.114 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:22 smithi090 ceph-mon[40003]: pgmap v46: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-12T08:51:23.114 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:22 smithi090 ceph-mon[95814]: pgmap v46: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-12T08:51:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:22 smithi114 ceph-mon[33573]: pgmap v46: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-12T08:51:23.414 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:51:23] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.43.0" 2024-04-12T08:51:23.452 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-12T08:51:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:51:24.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:24 smithi090 ceph-mon[95814]: pgmap v47: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-12T08:51:24.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:24 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:24.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:24 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:24.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:24 smithi090 ceph-mon[40003]: pgmap v47: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-12T08:51:24.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:24 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:24.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:24 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:25.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:24 smithi114 ceph-mon[33573]: pgmap v47: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-12T08:51:25.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:24 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:25.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:24 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:26.684 INFO:teuthology.orchestra.run.smithi090.stdout:wait for majority of mons to upgrade 2024-04-12T08:51:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:26 smithi090 ceph-mon[40003]: pgmap v48: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-12T08:51:26.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:26 smithi090 ceph-mon[95814]: pgmap v48: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-12T08:51:27.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:26 smithi114 ceph-mon[33573]: pgmap v48: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-12T08:51:28.038 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-12T08:51:28.688 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:28 smithi090 ceph-mon[40003]: pgmap v49: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:51:28.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:28 smithi090 ceph-mon[95814]: pgmap v49: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:51:28.964 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:51:29.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:51:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:51:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:51:29.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:28 smithi114 ceph-mon[33573]: pgmap v49: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:51:30.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:30 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:30.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:30 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:30.388 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:30 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:30.388 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:30 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:30.388 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:30 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:30.389 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:30 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[95814]: pgmap v50: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-04-12T08:51:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:51:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:51:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[40003]: pgmap v50: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T08:51:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:31 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-12T08:51:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:31 smithi114 ceph-mon[33573]: pgmap v50: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-04-12T08:51:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:31 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:31 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:51:31.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:31 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:51:31.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:31 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:31.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:31 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:51:31.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:31 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:51:31.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:31 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T08:51:31.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:31 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-12T08:51:32.406 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:32 smithi090 ceph-mon[40003]: pgmap v51: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-04-12T08:51:32.406 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:32 smithi090 ceph-mon[40003]: Upgrade: It appears safe to stop mon.c 2024-04-12T08:51:32.406 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:32 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/4023023110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:51:32.407 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:32 smithi090 ceph-mon[95814]: pgmap v51: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-04-12T08:51:32.407 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:32 smithi090 ceph-mon[95814]: Upgrade: It appears safe to stop mon.c 2024-04-12T08:51:32.407 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:32 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4023023110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:51:32.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:32 smithi114 ceph-mon[33573]: pgmap v51: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-04-12T08:51:32.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:32 smithi114 ceph-mon[33573]: Upgrade: It appears safe to stop mon.c 2024-04-12T08:51:32.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:32 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/4023023110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:51:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:51:33] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.43.0" 2024-04-12T08:51:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:34 smithi114 ceph-mon[33573]: pgmap v52: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 277 B/s wr, 731 op/s 2024-04-12T08:51:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:34 smithi114 ceph-mon[33573]: Upgrade: Updating mon.c 2024-04-12T08:51:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:34 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:34.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:34 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:51:34.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:34 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:51:34.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:34 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:51:34.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:34 smithi114 ceph-mon[33573]: Deploying daemon mon.c on smithi090 2024-04-12T08:51:34.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:34 smithi114 ceph-mon[33573]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-12T08:51:34.373 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[40003]: pgmap v52: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 277 B/s wr, 731 op/s 2024-04-12T08:51:34.373 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[40003]: Upgrade: Updating mon.c 2024-04-12T08:51:34.373 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[40003]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:34.373 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:51:34.373 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:51:34.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[40003]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:51:34.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[40003]: Deploying daemon mon.c on smithi090 2024-04-12T08:51:34.374 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[40003]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-12T08:51:34.374 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[95814]: pgmap v52: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 277 B/s wr, 731 op/s 2024-04-12T08:51:34.374 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[95814]: Upgrade: Updating mon.c 2024-04-12T08:51:34.374 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:34.374 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:51:34.374 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:51:34.374 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:51:34.375 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[95814]: Deploying daemon mon.c on smithi090 2024-04-12T08:51:34.375 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:34 smithi090 ceph-mon[95814]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-12T08:51:35.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:35 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3604252215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:51:35.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:35 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3604252215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:51:35.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:35 smithi090 ceph-mon[40003]: from='client.? 172.21.15.90:0/3604252215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:51:35.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:35 smithi090 systemd[1]: Stopping Ceph mon.c for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:51:36.324 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:51:36] ENGINE Bus STOPPING 2024-04-12T08:51:36.324 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:51:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T08:51:36.324 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:51:36] ENGINE Bus STOPPED 2024-04-12T08:51:36.324 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:51:36] ENGINE Bus STARTING 2024-04-12T08:51:36.324 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:51:36] ENGINE Serving on http://:::9283 2024-04-12T08:51:36.325 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:51:36] ENGINE Bus STARTED 2024-04-12T08:51:36.325 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c[39997]: 2024-04-12T08:51:36.062+0000 7f0dcfa2c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T08:51:36.325 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:36 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c[39997]: 2024-04-12T08:51:36.062+0000 7f0dcfa2c700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-12T08:51:36.325 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:36 smithi090 podman[99392]: 2024-04-12 08:51:36.118558912 +0000 UTC m=+0.208652308 container died d5aa934587fe7593704ae36a833843ee78f09f16def743564558edcb3e58d193 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=quincy-d3b5060, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-12T08:51:36.578 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:36 smithi090 podman[99392]: 2024-04-12 08:51:36.548900556 +0000 UTC m=+0.638993947 container cleanup d5aa934587fe7593704ae36a833843ee78f09f16def743564558edcb3e58d193 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, ceph=True, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T08:51:36.578 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:36 smithi090 bash[99392]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c 2024-04-12T08:51:37.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:36 smithi090 podman[99413]: 2024-04-12 08:51:36.851572064 +0000 UTC m=+0.725748417 container remove d5aa934587fe7593704ae36a833843ee78f09f16def743564558edcb3e58d193 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS) 2024-04-12T08:51:37.603 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:37 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-12T08:51:37.603 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:37 smithi090 systemd[1]: Stopped Ceph mon.c for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:51:37.603 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:37 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.c.service: Consumed 9.212s CPU time. 2024-04-12T08:51:37.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:37 smithi090 systemd[1]: Starting Ceph mon.c for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:51:38.282 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:37 smithi090 podman[99560]: 2024-04-12 08:51:37.865953109 +0000 UTC m=+0.051368724 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:51:38.282 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 podman[99560]: 2024-04-12 08:51:38.011610759 +0000 UTC m=+0.197026364 container create 6bfdbaae0edc6056f130989a9a37f3cac45c28369a63ad9679b9977294efe125 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-12T08:51:38.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 podman[99560]: 2024-04-12 08:51:38.282161624 +0000 UTC m=+0.467577257 container init 6bfdbaae0edc6056f130989a9a37f3cac45c28369a63ad9679b9977294efe125 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-12T08:51:38.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 podman[99560]: 2024-04-12 08:51:38.287438307 +0000 UTC m=+0.472853917 container start 6bfdbaae0edc6056f130989a9a37f3cac45c28369a63ad9679b9977294efe125 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-12T08:51:38.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T08:51:38.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev), process ceph-mon, pid 2 2024-04-12T08:51:38.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: pidfile_write: ignore empty --pid-file 2024-04-12T08:51:38.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: load: jerasure load: lrc 2024-04-12T08:51:38.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: RocksDB version: 7.9.2 2024-04-12T08:51:38.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Git sha 0 2024-04-12T08:51:38.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Compile date 2024-04-09 21:18:54 2024-04-12T08:51:38.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: DB SUMMARY 2024-04-12T08:51:38.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: DB Session ID: 1N1ZNSSUPZVR825OEVNC 2024-04-12T08:51:38.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: CURRENT file: CURRENT 2024-04-12T08:51:38.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T08:51:38.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1104 Bytes 2024-04-12T08:51:38.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-04-12T08:51:38.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 8261306 ; 2024-04-12T08:51:38.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.error_if_exists: 0 2024-04-12T08:51:38.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.create_if_missing: 0 2024-04-12T08:51:38.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.paranoid_checks: 1 2024-04-12T08:51:38.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T08:51:38.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T08:51:38.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T08:51:38.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.env: 0x560ac83c4c60 2024-04-12T08:51:38.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T08:51:38.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.info_log: 0x560ac900bae0 2024-04-12T08:51:38.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T08:51:38.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.statistics: (nil) 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.use_fsync: 0 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_log_file_size: 0 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.allow_fallocate: 1 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.use_direct_reads: 0 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.db_log_dir: 2024-04-12T08:51:38.679 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.wal_dir: 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.write_buffer_manager: 0x560ac8fde820 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T08:51:38.680 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.unordered_write: 0 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.row_cache: None 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.wal_filter: None 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.two_write_queues: 0 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.wal_compression: 0 2024-04-12T08:51:38.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.atomic_flush: 0 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.log_readahead_size: 0 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_background_jobs: 2 2024-04-12T08:51:38.682 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_background_compactions: -1 2024-04-12T08:51:38.683 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_subcompactions: 1 2024-04-12T08:51:38.683 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T08:51:38.683 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T08:51:38.683 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T08:51:38.683 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T08:51:38.683 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_open_files: -1 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_background_flushes: -1 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Compression algorithms supported: 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: kZSTD supported: 0 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: kXpressCompression supported: 0 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T08:51:38.684 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: kZlibCompression supported: 1 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: kSnappyCompression supported: 1 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: kLZ4Compression supported: 1 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: kBZip2Compression supported: 0 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.merge_operator: 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_filter: None 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T08:51:38.685 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560ac900bbe0) 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: cache_index_and_filter_blocks: 1 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: pin_top_level_index_and_filter: 1 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: index_type: 0 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: data_block_index_type: 0 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: index_shortening: 1 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: checksum: 4 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: no_block_cache: 0 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_cache: 0x560ac900f090 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_cache_name: BinnedLRUCache 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_cache_options: 2024-04-12T08:51:38.686 INFO:journalctl@ceph.mon.c.smithi090.stdout: capacity : 536870912 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: num_shard_bits : 4 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: strict_capacity_limit : 0 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: high_pri_pool_ratio: 0.000 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_cache_compressed: (nil) 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: persistent_cache: (nil) 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_size: 4096 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_size_deviation: 10 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_restart_interval: 16 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: index_block_restart_interval: 1 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: metadata_block_size: 4096 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: partition_filters: 0 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: use_delta_encoding: 1 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: filter_policy: bloomfilter 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: whole_key_filtering: 1 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: verify_compression: 0 2024-04-12T08:51:38.687 INFO:journalctl@ceph.mon.c.smithi090.stdout: read_amp_bytes_per_bit: 0 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout: format_version: 5 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout: enable_index_compression: 1 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout: block_align: 0 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout: max_auto_readahead_size: 262144 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout: prepopulate_block_cache: 0 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout: initial_auto_readahead_size: 8192 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compression: NoCompression 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.num_levels: 7 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T08:51:38.688 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T08:51:38.689 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T08:51:38.690 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T08:51:38.690 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T08:51:38.690 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T08:51:38.690 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T08:51:38.690 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T08:51:38.690 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T08:51:38.690 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T08:51:38.690 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T08:51:38.691 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T08:51:38.692 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.table_properties_collectors: 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.inplace_update_support: 0 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.bloom_locality: 0 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.max_successive_merges: 0 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T08:51:38.693 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.ttl: 2592000 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.enable_blob_files: false 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.min_blob_size: 0 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T08:51:38.694 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T08:51:38.695 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T08:51:38.695 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T08:51:38.695 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T08:51:38.695 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T08:51:38.695 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T08:51:38.695 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-12T08:51:38.695 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T08:51:38.695 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12628, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T08:51:38.695 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 747a31ab-5eb5-4e39-9784-9ca0f00de54d 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911898346308, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911898373549, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 5075707, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12633, "largest_seqno": 15557, "table_properties": {"data_size": 5063712, "index_size": 7710, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3269, "raw_key_size": 27741, "raw_average_key_size": 21, "raw_value_size": 5037899, "raw_average_value_size": 3884, "num_data_blocks": 347, "num_entries": 1297, "num_filter_entries": 1297, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712911898, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "747a31ab-5eb5-4e39-9784-9ca0f00de54d", "db_session_id": "1N1ZNSSUPZVR825OEVNC", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911898373656, "job": 1, "event": "recovery_finished"} 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560ac9100000 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: DB pointer 0x560ac90ea000 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** DB Stats ** 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:51:38.696 INFO:journalctl@ceph.mon.c.smithi090.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T08:51:38.697 INFO:journalctl@ceph.mon.c.smithi090.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T08:51:38.697 INFO:journalctl@ceph.mon.c.smithi090.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T08:51:38.697 INFO:journalctl@ceph.mon.c.smithi090.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T08:51:38.697 INFO:journalctl@ceph.mon.c.smithi090.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T08:51:38.697 INFO:journalctl@ceph.mon.c.smithi090.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T08:51:38.697 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:51:38.697 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:51:38.697 INFO:journalctl@ceph.mon.c.smithi090.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T08:51:38.698 INFO:journalctl@ceph.mon.c.smithi090.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T08:51:38.698 INFO:journalctl@ceph.mon.c.smithi090.stdout: L0 1/0 4.84 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 450.5 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-12T08:51:38.698 INFO:journalctl@ceph.mon.c.smithi090.stdout: L6 1/0 6.05 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-12T08:51:38.698 INFO:journalctl@ceph.mon.c.smithi090.stdout: Sum 2/0 10.89 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 450.5 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-12T08:51:38.698 INFO:journalctl@ceph.mon.c.smithi090.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 450.5 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-12T08:51:38.698 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:51:38.699 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-12T08:51:38.699 INFO:journalctl@ceph.mon.c.smithi090.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T08:51:38.699 INFO:journalctl@ceph.mon.c.smithi090.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:51:38.699 INFO:journalctl@ceph.mon.c.smithi090.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 450.5 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: Cumulative compaction: 0.00 GB write, 129.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:51:38.700 INFO:journalctl@ceph.mon.c.smithi090.stdout: Interval compaction: 0.00 GB write, 129.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:51:38.701 INFO:journalctl@ceph.mon.c.smithi090.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T08:51:38.701 INFO:journalctl@ceph.mon.c.smithi090.stdout: Block cache BinnedLRUCache@0x560ac900f090#2 capacity: 512.00 MB usage: 53.20 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-04-12T08:51:38.701 INFO:journalctl@ceph.mon.c.smithi090.stdout: Block cache entry stats(count,size,portion): DataBlock(3,16.73 KB,0.00319183%) FilterBlock(2,12.31 KB,0.00234842%) IndexBlock(2,24.16 KB,0.00460744%) Misc(1,0.00 KB,0%) 2024-04-12T08:51:38.701 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:51:38.701 INFO:journalctl@ceph.mon.c.smithi090.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T08:51:38.701 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: starting mon.c rank 2 at public addrs [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] at bind addrs [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:51:38.702 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???) e3 preinit fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:51:38.702 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???).mds e8 new map 2024-04-12T08:51:38.702 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???).mds e8 print_map 2024-04-12T08:51:38.702 INFO:journalctl@ceph.mon.c.smithi090.stdout: e8 2024-04-12T08:51:38.702 INFO:journalctl@ceph.mon.c.smithi090.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T08:51:38.702 INFO:journalctl@ceph.mon.c.smithi090.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T08:51:38.702 INFO:journalctl@ceph.mon.c.smithi090.stdout: legacy client fscid: 1 2024-04-12T08:51:38.702 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:51:38.702 INFO:journalctl@ceph.mon.c.smithi090.stdout: Filesystem 'foo' (1) 2024-04-12T08:51:38.702 INFO:journalctl@ceph.mon.c.smithi090.stdout: fs_name foo 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: epoch 7 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: created 2024-04-12T08:43:28.135965+0000 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: modified 2024-04-12T08:43:37.934695+0000 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: tableserver 0 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: root 0 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: session_timeout 60 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: session_autoclose 300 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: max_file_size 1099511627776 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: max_xattr_size 65536 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: required_client_features {} 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: last_failure 0 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: last_failure_osd_epoch 0 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T08:51:38.703 INFO:journalctl@ceph.mon.c.smithi090.stdout: max_mds 1 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: in 0 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: up {0=14631} 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: failed 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: damaged 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: stopped 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: data_pools [4] 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: metadata_pool 3 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: inline_data disabled 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: balancer 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: bal_rank_mask -1 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: standby_count_wanted 1 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: qdb_cluster leader: 0 members: 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: [mds.foo.smithi090.rxdrqj{0:14631} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:51:38.704 INFO:journalctl@ceph.mon.c.smithi090.stdout: Standby daemons: 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout: 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout: [mds.foo.smithi114.nbzjob{-1:24496} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.114:6832/562045409,v1:172.21.15.114:6833/562045409] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???).mgr e0 loading version 39 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???).mgr e39 active server: [v2:172.21.15.90:6800/1467705464,v1:172.21.15.90:6801/1467705464](34104) 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 ceph-mon[99575]: mon.c@-1(???).mgr e39 mkfs or daemon transitioned to available, loading commands 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 bash[99560]: 6bfdbaae0edc6056f130989a9a37f3cac45c28369a63ad9679b9977294efe125 2024-04-12T08:51:38.705 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:38 smithi090 systemd[1]: Started Ceph mon.c for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:51:39.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:51:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:51:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:51:43.621 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:51:43] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.43.0" 2024-04-12T08:51:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:43 smithi090 ceph-mon[95814]: pgmap v53: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 277 B/s wr, 731 op/s 2024-04-12T08:51:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:43 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:51:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:43 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:51:43.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:43 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:51:44.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:43 smithi114 ceph-mon[33573]: pgmap v53: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 277 B/s wr, 731 op/s 2024-04-12T08:51:44.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:43 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:51:44.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:43 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:51:44.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:43 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:51:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: mon.c calling monitor election 2024-04-12T08:51:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: mon.b calling monitor election 2024-04-12T08:51:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: mon.a calling monitor election 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: monmap epoch 3 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: last_changed 2024-04-12T08:38:07.849098+0000 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: created 2024-04-12T08:36:31.874980+0000 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: min_mon_release 17 (quincy) 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: election_strategy: 1 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: 0: [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon.a 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: 1: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.b 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: 2: [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon.c 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: mgrmap e39: y(active, since 104s), standbys: x 2024-04-12T08:51:44.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-12T08:51:44.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:51:44.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T08:51:44.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[95814]: daemon grafana.a on smithi114 is in error state 2024-04-12T08:51:44.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: pgmap v53: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 277 B/s wr, 731 op/s 2024-04-12T08:51:44.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:51:44.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:51:44.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:51:44.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: mon.c calling monitor election 2024-04-12T08:51:44.925 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: mon.b calling monitor election 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: mon.a calling monitor election 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: monmap epoch 3 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: last_changed 2024-04-12T08:38:07.849098+0000 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: created 2024-04-12T08:36:31.874980+0000 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: min_mon_release 17 (quincy) 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: election_strategy: 1 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: 0: [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon.a 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: 1: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.b 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: 2: [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon.c 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:51:44.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T08:51:44.927 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: mgrmap e39: y(active, since 104s), standbys: x 2024-04-12T08:51:44.927 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-12T08:51:44.927 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:51:44.927 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T08:51:44.927 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:44 smithi090 ceph-mon[99575]: daemon grafana.a on smithi114 is in error state 2024-04-12T08:51:45.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: mon.c calling monitor election 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: mon.b calling monitor election 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: mon.a calling monitor election 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: monmap epoch 3 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: last_changed 2024-04-12T08:38:07.849098+0000 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: created 2024-04-12T08:36:31.874980+0000 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: min_mon_release 17 (quincy) 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: election_strategy: 1 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: 0: [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon.a 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: 1: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.b 2024-04-12T08:51:45.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: 2: [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon.c 2024-04-12T08:51:45.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:51:45.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T08:51:45.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: mgrmap e39: y(active, since 104s), standbys: x 2024-04-12T08:51:45.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-12T08:51:45.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:51:45.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T08:51:45.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:44 smithi114 ceph-mon[33573]: daemon grafana.a on smithi114 is in error state 2024-04-12T08:51:46.017 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:45 smithi114 ceph-mon[33573]: pgmap v58: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-12T08:51:46.018 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:45 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:46.129 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:45 smithi090 ceph-mon[95814]: pgmap v58: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-12T08:51:46.129 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:46.129 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:45 smithi090 ceph-mon[99575]: pgmap v58: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-12T08:51:46.129 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:51:47.193 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T08:51:47.859 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:47 smithi090 ceph-mon[99575]: pgmap v59: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T08:51:47.859 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:47 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3944082454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:51:47.859 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:47 smithi090 ceph-mon[95814]: pgmap v59: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T08:51:47.860 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:47 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3944082454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:51:47.896 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch ps' 2024-04-12T08:51:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:47 smithi114 ceph-mon[33573]: pgmap v59: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T08:51:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:47 smithi114 ceph-mon[33573]: from='client.? 172.21.15.90:0/3944082454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:51:48.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:48 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:51:48.397+0000 7efe43a91700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-12T08:51:49.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:51:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:51:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:51:50.122 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:51:50.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:50 smithi090 ceph-mon[99575]: pgmap v60: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:51:50.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:50 smithi090 ceph-mon[95814]: pgmap v60: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:51:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:50 smithi114 ceph-mon[33573]: pgmap v60: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:51:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:51 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:51 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:51 smithi090 ceph-mon[99575]: pgmap v61: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:51:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:51 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:51 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:51 smithi090 ceph-mon[95814]: pgmap v61: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:51:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:51 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:51 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:51 smithi114 ceph-mon[33573]: pgmap v61: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-12T08:51:53.348 INFO:teuthology.orchestra.run.smithi090.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T08:51:53.348 INFO:teuthology.orchestra.run.smithi090.stdout:alertmanager.a smithi090 *:9093,9094 running (4m) 2s ago 9m 25.4M - 0.25.0 c8568f914cd2 d11ba2dc752a 2024-04-12T08:51:53.348 INFO:teuthology.orchestra.run.smithi090.stdout:grafana.a smithi114 *:3000 error 29s ago 9m - - 2024-04-12T08:51:53.348 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi090.rxdrqj smithi090 running (8m) 2s ago 8m 19.4M - 17.2.7-827-gd3b5060a 21de9b7369ad f627bce7c272 2024-04-12T08:51:53.348 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi114.nbzjob smithi114 running (8m) 29s ago 8m 14.2M - 17.2.7-827-gd3b5060a 21de9b7369ad 60cb65499580 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.x smithi114 *:8443,9283,8765 running (2m) 29s ago 13m 449M - 19.0.0-2053-gb277feb2 668895944f0e 60c0ca408f91 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.y smithi090 *:8443,9283,8765 running (3m) 2s ago 15m 499M - 19.0.0-2053-gb277feb2 668895944f0e b61a0f582c61 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:mon.a smithi090 running (2m) 2s ago 15m 41.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e ad8458aaf579 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:mon.b smithi114 running (13m) 29s ago 13m 50.3M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 63cd6193074e 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:mon.c smithi090 running (15s) 2s ago 13m 27.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e 6bfdbaae0edc 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.a smithi090 *:9100 running (3m) 2s ago 9m 16.8M - 1.5.0 0da6a335fe13 687747c6880c 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.b smithi114 *:9100 running (3m) 29s ago 9m 15.4M - 1.5.0 0da6a335fe13 e89762981b55 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:osd.0 smithi090 running (13m) 2s ago 13m 374M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 9395ed639a37 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:osd.1 smithi090 running (12m) 2s ago 12m 335M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 98eb8b5854e9 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:osd.2 smithi090 running (12m) 2s ago 12m 464M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 8008248b4e61 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:osd.3 smithi090 running (11m) 2s ago 11m 483M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 83a5641ba19a 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:osd.4 smithi114 running (11m) 29s ago 11m 404M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 15d0d8563392 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:osd.5 smithi114 running (11m) 29s ago 11m 433M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 54049e134882 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:osd.6 smithi114 running (10m) 29s ago 10m 297M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 86dfedc8629b 2024-04-12T08:51:53.349 INFO:teuthology.orchestra.run.smithi090.stdout:osd.7 smithi114 running (10m) 29s ago 10m 377M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad dda597b19712 2024-04-12T08:51:53.350 INFO:teuthology.orchestra.run.smithi090.stdout:prometheus.a smithi114 *:9095 running (2m) 29s ago 9m 61.3M - 2.43.0 a07b618ecd1d f009118ca428 2024-04-12T08:51:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:51:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:51:53] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.43.0" 2024-04-12T08:51:54.354 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade pause' 2024-04-12T08:51:54.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:54 smithi090 ceph-mon[95814]: pgmap v62: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2024-04-12T08:51:54.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:54 smithi090 ceph-mon[95814]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:51:54.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:54 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:54.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:54 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:54.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:54 smithi090 ceph-mon[99575]: pgmap v62: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2024-04-12T08:51:54.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:54 smithi090 ceph-mon[99575]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:51:54.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:54 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:54.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:54 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:54 smithi114 ceph-mon[33573]: pgmap v62: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2024-04-12T08:51:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:54 smithi114 ceph-mon[33573]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:51:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:54 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:54 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:51:55.495 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:51:56.627 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:56 smithi090 ceph-mon[95814]: pgmap v63: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-04-12T08:51:56.627 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:56 smithi090 ceph-mon[99575]: pgmap v63: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-04-12T08:51:56.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:56 smithi114 ceph-mon[33573]: pgmap v63: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-04-12T08:51:58.636 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:51:58 smithi090 ceph-mon[99575]: pgmap v64: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2024-04-12T08:51:58.637 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:51:58 smithi090 ceph-mon[95814]: pgmap v64: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2024-04-12T08:51:58.660 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:51:58 smithi114 ceph-mon[33573]: pgmap v64: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2024-04-12T08:51:59.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:51:58 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:51:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:51:59.877 INFO:teuthology.orchestra.run.smithi090.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:52:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:00 smithi090 ceph-mon[99575]: pgmap v65: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-04-12T08:52:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:00.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:00 smithi090 ceph-mon[95814]: pgmap v65: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-04-12T08:52:00.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:00 smithi114 ceph-mon[33573]: pgmap v65: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-04-12T08:52:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:00 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.151 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'sleep 60' 2024-04-12T08:52:01.622 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:52:01.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:52:01.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:52:01.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:52:01.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:52:01.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: pgmap v66: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2024-04-12T08:52:01.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: pgmap v67: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-04-12T08:52:01.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:01.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: pgmap v66: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: pgmap v67: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-04-12T08:52:01.773 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:01.774 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.774 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:52:01.774 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:52:01.774 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:52:01.774 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:52:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:52:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:52:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:52:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: pgmap v66: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2024-04-12T08:52:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: pgmap v67: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-04-12T08:52:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:52:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:52:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:52:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:01 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:52:02.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:02 smithi090 ceph-mon[95814]: pgmap v68: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 501 op/s 2024-04-12T08:52:02.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:02 smithi090 ceph-mon[95814]: pgmap v69: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-12T08:52:02.828 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:02 smithi090 ceph-mon[99575]: pgmap v68: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 501 op/s 2024-04-12T08:52:02.829 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:02 smithi090 ceph-mon[99575]: pgmap v69: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-12T08:52:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:02 smithi114 ceph-mon[33573]: pgmap v68: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 501 op/s 2024-04-12T08:52:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:02 smithi114 ceph-mon[33573]: pgmap v69: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-12T08:52:03.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:52:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:52:03] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.43.0" 2024-04-12T08:52:04.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:04 smithi114 ceph-mon[33573]: pgmap v70: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-04-12T08:52:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:04 smithi090 ceph-mon[95814]: pgmap v70: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-04-12T08:52:04.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:04 smithi090 ceph-mon[99575]: pgmap v70: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-04-12T08:52:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:06 smithi114 ceph-mon[33573]: pgmap v71: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2024-04-12T08:52:06.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:06 smithi090 ceph-mon[95814]: pgmap v71: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2024-04-12T08:52:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:06 smithi090 ceph-mon[99575]: pgmap v71: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2024-04-12T08:52:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:08 smithi114 ceph-mon[33573]: pgmap v72: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.23k op/s 2024-04-12T08:52:08.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:52:08 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:52:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:52:08.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:08 smithi090 ceph-mon[95814]: pgmap v72: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.23k op/s 2024-04-12T08:52:08.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:08 smithi090 ceph-mon[99575]: pgmap v72: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.23k op/s 2024-04-12T08:52:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:10 smithi114 ceph-mon[33573]: pgmap v73: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:52:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:10 smithi090 ceph-mon[95814]: pgmap v73: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:52:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:10 smithi090 ceph-mon[99575]: pgmap v73: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:52:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:12 smithi114 ceph-mon[33573]: pgmap v74: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2024-04-12T08:52:12.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:12 smithi090 ceph-mon[99575]: pgmap v74: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2024-04-12T08:52:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:12 smithi090 ceph-mon[95814]: pgmap v74: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2024-04-12T08:52:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:52:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:52:13] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.43.0" 2024-04-12T08:52:14.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:14 smithi114 ceph-mon[33573]: pgmap v75: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-12T08:52:14.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:14 smithi090 ceph-mon[99575]: pgmap v75: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-12T08:52:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:14 smithi090 ceph-mon[95814]: pgmap v75: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-12T08:52:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:15 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:15 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:16.768 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:16 smithi114 ceph-mon[33573]: pgmap v76: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2024-04-12T08:52:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:16 smithi090 ceph-mon[95814]: pgmap v76: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2024-04-12T08:52:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:16 smithi090 ceph-mon[99575]: pgmap v76: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2024-04-12T08:52:18.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:18 smithi114 ceph-mon[33573]: pgmap v77: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T08:52:18.870 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:52:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:52:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:52:18.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:18 smithi090 ceph-mon[95814]: pgmap v77: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T08:52:18.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:18 smithi090 ceph-mon[99575]: pgmap v77: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T08:52:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:20 smithi114 ceph-mon[33573]: pgmap v78: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 810 op/s 2024-04-12T08:52:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:20 smithi090 ceph-mon[95814]: pgmap v78: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 810 op/s 2024-04-12T08:52:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:20 smithi090 ceph-mon[99575]: pgmap v78: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 810 op/s 2024-04-12T08:52:22.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:22 smithi090 ceph-mon[95814]: pgmap v79: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-04-12T08:52:22.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:22 smithi090 ceph-mon[99575]: pgmap v79: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-04-12T08:52:22.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:22 smithi114 ceph-mon[33573]: pgmap v79: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-04-12T08:52:23.619 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:52:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:52:23] "GET /metrics HTTP/1.1" 200 36280 "" "Prometheus/2.43.0" 2024-04-12T08:52:24.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:24 smithi114 ceph-mon[33573]: pgmap v80: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-04-12T08:52:24.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:24 smithi090 ceph-mon[95814]: pgmap v80: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-04-12T08:52:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:24 smithi090 ceph-mon[99575]: pgmap v80: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-04-12T08:52:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:26 smithi114 ceph-mon[33573]: pgmap v81: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-04-12T08:52:26.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:26 smithi090 ceph-mon[95814]: pgmap v81: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-04-12T08:52:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:26 smithi090 ceph-mon[99575]: pgmap v81: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-04-12T08:52:28.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:52:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:52:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:52:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:28 smithi114 ceph-mon[33573]: pgmap v82: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:52:28.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:28 smithi090 ceph-mon[95814]: pgmap v82: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:52:28.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:28 smithi090 ceph-mon[99575]: pgmap v82: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-12T08:52:30.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:30 smithi114 ceph-mon[33573]: pgmap v83: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-04-12T08:52:30.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:30 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:30.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:30 smithi090 ceph-mon[95814]: pgmap v83: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-04-12T08:52:30.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:30 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:30.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:30 smithi090 ceph-mon[99575]: pgmap v83: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-04-12T08:52:30.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:30 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:32.829 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:32 smithi090 ceph-mon[95814]: pgmap v84: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-12T08:52:32.829 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:32 smithi090 ceph-mon[99575]: pgmap v84: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-12T08:52:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:32 smithi114 ceph-mon[33573]: pgmap v84: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-12T08:52:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:52:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:52:33] "GET /metrics HTTP/1.1" 200 36276 "" "Prometheus/2.43.0" 2024-04-12T08:52:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:34 smithi114 ceph-mon[33573]: pgmap v85: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-04-12T08:52:34.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:34 smithi090 ceph-mon[95814]: pgmap v85: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-04-12T08:52:34.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:34 smithi090 ceph-mon[99575]: pgmap v85: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-04-12T08:52:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:36 smithi114 ceph-mon[33573]: pgmap v86: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-04-12T08:52:36.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:36 smithi090 ceph-mon[95814]: pgmap v86: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-04-12T08:52:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:36 smithi090 ceph-mon[99575]: pgmap v86: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-04-12T08:52:38.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:38 smithi114 ceph-mon[33573]: pgmap v87: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:52:38.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:52:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:52:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:52:38.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:38 smithi090 ceph-mon[95814]: pgmap v87: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:52:38.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:38 smithi090 ceph-mon[99575]: pgmap v87: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-12T08:52:40.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:40 smithi114 ceph-mon[33573]: pgmap v88: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-04-12T08:52:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:40 smithi090 ceph-mon[95814]: pgmap v88: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-04-12T08:52:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:40 smithi090 ceph-mon[99575]: pgmap v88: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-04-12T08:52:42.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:42 smithi114 ceph-mon[33573]: pgmap v89: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2024-04-12T08:52:42.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:42 smithi090 ceph-mon[95814]: pgmap v89: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2024-04-12T08:52:42.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:42 smithi090 ceph-mon[99575]: pgmap v89: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2024-04-12T08:52:43.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:52:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:52:43] "GET /metrics HTTP/1.1" 200 36276 "" "Prometheus/2.43.0" 2024-04-12T08:52:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:44 smithi114 ceph-mon[33573]: pgmap v90: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-04-12T08:52:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:44 smithi090 ceph-mon[99575]: pgmap v90: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-04-12T08:52:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:44 smithi090 ceph-mon[95814]: pgmap v90: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-04-12T08:52:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:45 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:52:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:46 smithi114 ceph-mon[33573]: pgmap v91: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-04-12T08:52:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:46 smithi090 ceph-mon[95814]: pgmap v91: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-04-12T08:52:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:46 smithi090 ceph-mon[99575]: pgmap v91: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-04-12T08:52:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:47 smithi114 ceph-mon[33573]: pgmap v92: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:52:47.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:47 smithi090 ceph-mon[95814]: pgmap v92: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:52:47.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:47 smithi090 ceph-mon[99575]: pgmap v92: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:52:49.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 08:52:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T08:52:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi114.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.90\", device_class=\"ssd\", hostname=\"smithi090\", instance=\"smithi090:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.90\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T08:52:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:49 smithi114 ceph-mon[33573]: pgmap v93: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-04-12T08:52:50.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:49 smithi090 ceph-mon[99575]: pgmap v93: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-04-12T08:52:50.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:49 smithi090 ceph-mon[95814]: pgmap v93: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-04-12T08:52:51.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:51 smithi090 ceph-mon[99575]: pgmap v94: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-04-12T08:52:51.848 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:51 smithi090 ceph-mon[95814]: pgmap v94: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-04-12T08:52:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:51 smithi114 ceph-mon[33573]: pgmap v94: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-04-12T08:52:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:52:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:52:53] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.43.0" 2024-04-12T08:52:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:53 smithi114 ceph-mon[33573]: pgmap v95: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-04-12T08:52:54.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:53 smithi090 ceph-mon[95814]: pgmap v95: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-04-12T08:52:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:53 smithi090 ceph-mon[99575]: pgmap v95: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-04-12T08:52:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:55 smithi114 ceph-mon[33573]: pgmap v96: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-04-12T08:52:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:55 smithi090 ceph-mon[95814]: pgmap v96: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-04-12T08:52:56.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:55 smithi090 ceph-mon[99575]: pgmap v96: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-04-12T08:52:58.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:57 smithi114 ceph-mon[33573]: pgmap v97: 89 pgs: 89 active+clean; 901 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-12T08:52:58.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:57 smithi090 ceph-mon[95814]: pgmap v97: 89 pgs: 89 active+clean; 901 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-12T08:52:58.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:57 smithi090 ceph-mon[99575]: pgmap v97: 89 pgs: 89 active+clean; 901 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-12T08:53:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:52:59 smithi114 ceph-mon[33573]: pgmap v98: 89 pgs: 89 active+clean; 838 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-04-12T08:53:00.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:52:59 smithi090 ceph-mon[99575]: pgmap v98: 89 pgs: 89 active+clean; 838 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-04-12T08:53:00.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:52:59 smithi090 ceph-mon[95814]: pgmap v98: 89 pgs: 89 active+clean; 838 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-04-12T08:53:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:00 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:02.049 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:01 smithi114 ceph-mon[33573]: pgmap v99: 89 pgs: 89 active+clean; 838 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-12T08:53:02.049 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:01 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:53:02.141 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:01 smithi090 ceph-mon[99575]: pgmap v99: 89 pgs: 89 active+clean; 838 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-12T08:53:02.141 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:53:02.141 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:01 smithi090 ceph-mon[95814]: pgmap v99: 89 pgs: 89 active+clean; 838 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-12T08:53:02.142 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:53:03.371 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:53:03] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.43.0" 2024-04-12T08:53:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:03 smithi090 ceph-mon[99575]: pgmap v100: 89 pgs: 89 active+clean; 566 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-12T08:53:03.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:03 smithi090 ceph-mon[95814]: pgmap v100: 89 pgs: 89 active+clean; 566 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-12T08:53:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:03 smithi114 ceph-mon[33573]: pgmap v100: 89 pgs: 89 active+clean; 566 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-12T08:53:05.153 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade resume' 2024-04-12T08:53:05.615 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:53:06.109 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:05 smithi090 ceph-mon[95814]: pgmap v101: 89 pgs: 89 active+clean; 566 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-04-12T08:53:06.109 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:05 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:06.109 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:05 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:06.109 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:05 smithi090 ceph-mon[99575]: pgmap v101: 89 pgs: 89 active+clean; 566 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-04-12T08:53:06.109 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:05 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:06.110 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:05 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:06.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:05 smithi114 ceph-mon[33573]: pgmap v101: 89 pgs: 89 active+clean; 566 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-04-12T08:53:06.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:05 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:06.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:05 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:07 smithi114 ceph-mon[33573]: pgmap v102: 89 pgs: 89 active+clean; 379 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2024-04-12T08:53:08.168 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:07 smithi090 ceph-mon[99575]: pgmap v102: 89 pgs: 89 active+clean; 379 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2024-04-12T08:53:08.169 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:07 smithi090 ceph-mon[95814]: pgmap v102: 89 pgs: 89 active+clean; 379 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2024-04-12T08:53:09.080 INFO:teuthology.orchestra.run.smithi090.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:53:10.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:10 smithi114 ceph-mon[33573]: pgmap v103: 89 pgs: 89 active+clean; 313 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-04-12T08:53:10.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:10 smithi114 ceph-mon[33573]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:53:10.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:10 smithi114 ceph-mon[33573]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:53:10.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:10 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:10.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:10 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:10.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:10 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:10.397 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[95814]: pgmap v103: 89 pgs: 89 active+clean; 313 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-04-12T08:53:10.397 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[95814]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:53:10.397 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[95814]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:53:10.397 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:10.397 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:10.397 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:10.397 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[99575]: pgmap v103: 89 pgs: 89 active+clean; 313 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-04-12T08:53:10.398 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[99575]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:53:10.398 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[99575]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:53:10.398 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:10.398 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:10.398 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:10 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:10.445 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'echo wait for all mons to upgrade' 2024-04-12T08:53:10.941 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:53:11.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:11 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:11.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:11 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:53:11.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:11 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:11.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:11 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:53:11.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:11 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:53:11.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:11 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T08:53:11.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:11 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-12T08:53:11.369 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:11.369 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:53:11.369 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:11.369 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:53:11.369 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:53:11.370 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T08:53:11.370 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-12T08:53:11.370 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:11.370 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:53:11.370 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:11.370 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:53:11.370 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:53:11.370 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T08:53:11.370 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-12T08:53:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:12 smithi114 ceph-mon[33573]: pgmap v104: 89 pgs: 89 active+clean; 313 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-12T08:53:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:12 smithi114 ceph-mon[33573]: Upgrade: It appears safe to stop mon.b 2024-04-12T08:53:12.403 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:12 smithi090 ceph-mon[99575]: pgmap v104: 89 pgs: 89 active+clean; 313 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-12T08:53:12.403 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:12 smithi090 ceph-mon[99575]: Upgrade: It appears safe to stop mon.b 2024-04-12T08:53:12.403 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:12 smithi090 ceph-mon[95814]: pgmap v104: 89 pgs: 89 active+clean; 313 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-12T08:53:12.403 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:12 smithi090 ceph-mon[95814]: Upgrade: It appears safe to stop mon.b 2024-04-12T08:53:12.798 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Removed 126675 objects 2024-04-12T08:53:12.799 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Clean up completed and total clean up time :146.154 2024-04-12T08:53:12.833 INFO:teuthology.orchestra.run.smithi090.stdout:wait for all mons to upgrade 2024-04-12T08:53:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:53:13] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.43.0" 2024-04-12T08:53:13.976 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-12T08:53:14.277 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:53:14.531 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-04-12T08:53:14.531 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-04-12T08:53:14.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:14 smithi114 ceph-mon[33573]: pgmap v105: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:53:14.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:14 smithi114 ceph-mon[33573]: Upgrade: Updating mon.b 2024-04-12T08:53:14.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:14 smithi114 ceph-mon[33573]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:14.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:14 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:14.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:14 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:14.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:14 smithi114 ceph-mon[33573]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:14.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:14 smithi114 ceph-mon[33573]: Deploying daemon mon.b on smithi114 2024-04-12T08:53:14.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[95814]: pgmap v105: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:53:14.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[95814]: Upgrade: Updating mon.b 2024-04-12T08:53:14.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:14.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:14.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:14.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:14.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[95814]: Deploying daemon mon.b on smithi114 2024-04-12T08:53:14.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[99575]: pgmap v105: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-12T08:53:14.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[99575]: Upgrade: Updating mon.b 2024-04-12T08:53:14.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:14.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:14.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:14.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:14.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:14 smithi090 ceph-mon[99575]: Deploying daemon mon.b on smithi114 2024-04-12T08:53:15.966 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:15 smithi114 systemd[1]: Stopping Ceph mon.b for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:53:16.258 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b[33563]: 2024-04-12T08:53:15.964+0000 7f3b67638700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T08:53:16.258 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b[33563]: 2024-04-12T08:53:15.964+0000 7f3b67638700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-12T08:53:16.258 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:15 smithi114 podman[89315]: 2024-04-12 08:53:15.985919903 +0000 UTC m=+0.147041344 container died 63cd6193074e5313de530327f860f7fb63f2038c129a15da91013e35fb6f4dc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, ceph=True) 2024-04-12T08:53:16.368 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:15 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:53:15] ENGINE Bus STOPPING 2024-04-12T08:53:16.369 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:15 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:53:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T08:53:16.369 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:15 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:53:15] ENGINE Bus STOPPED 2024-04-12T08:53:16.369 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:15 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:53:15] ENGINE Bus STARTING 2024-04-12T08:53:16.369 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:53:16] ENGINE Serving on http://:::9283 2024-04-12T08:53:16.369 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:08:53:16] ENGINE Bus STARTED 2024-04-12T08:53:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:16 smithi114 podman[89315]: 2024-04-12 08:53:16.35697454 +0000 UTC m=+0.518095987 container cleanup 63cd6193074e5313de530327f860f7fb63f2038c129a15da91013e35fb6f4dc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, ceph=True, org.label-schema.schema-version=1.0) 2024-04-12T08:53:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:16 smithi114 bash[89315]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b 2024-04-12T08:53:16.963 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:16 smithi114 podman[89329]: 2024-04-12 08:53:16.642063767 +0000 UTC m=+0.650647559 container remove 63cd6193074e5313de530327f860f7fb63f2038c129a15da91013e35fb6f4dc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409) 2024-04-12T08:53:16.963 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:16 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-12T08:53:16.963 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:16 smithi114 systemd[1]: Stopped Ceph mon.b for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:53:16.963 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:16 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.b.service: Consumed 11.844s CPU time. 2024-04-12T08:53:16.985 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:53:17.282 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:16 smithi114 systemd[1]: Starting Ceph mon.b for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:53:17.283 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 podman[89462]: 2024-04-12 08:53:17.07860973 +0000 UTC m=+0.017959548 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:53:17.283 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 podman[89462]: 2024-04-12 08:53:17.18479278 +0000 UTC m=+0.124142589 container create cd2de45bf1bf54bbe8467638c6a2d9ada8d431328156bed7f5547d646baa82c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=) 2024-04-12T08:53:17.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 podman[89462]: 2024-04-12 08:53:17.379253135 +0000 UTC m=+0.318602943 container init cd2de45bf1bf54bbe8467638c6a2d9ada8d431328156bed7f5547d646baa82c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T08:53:17.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 podman[89462]: 2024-04-12 08:53:17.382348665 +0000 UTC m=+0.321698482 container start cd2de45bf1bf54bbe8467638c6a2d9ada8d431328156bed7f5547d646baa82c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T08:53:17.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T08:53:17.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev), process ceph-mon, pid 2 2024-04-12T08:53:17.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: pidfile_write: ignore empty --pid-file 2024-04-12T08:53:17.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: load: jerasure load: lrc 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: RocksDB version: 7.9.2 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Git sha 0 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Compile date 2024-04-09 21:18:54 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: DB SUMMARY 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: DB Session ID: SARUYX3UBBCTJP0Q2811 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: CURRENT file: CURRENT 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1104 Bytes 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 9910441 ; 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.error_if_exists: 0 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.create_if_missing: 0 2024-04-12T08:53:17.619 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.paranoid_checks: 1 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.env: 0x556ec57b6c60 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.info_log: 0x556ec7c0fae0 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.statistics: (nil) 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.use_fsync: 0 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_log_file_size: 0 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T08:53:17.620 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.allow_fallocate: 1 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.use_direct_reads: 0 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.db_log_dir: 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.wal_dir: 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T08:53:17.621 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.write_buffer_manager: 0x556ec7be2820 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.unordered_write: 0 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T08:53:17.622 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.row_cache: None 2024-04-12T08:53:17.623 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.wal_filter: None 2024-04-12T08:53:17.623 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T08:53:17.624 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T08:53:17.624 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.two_write_queues: 0 2024-04-12T08:53:17.624 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T08:53:17.624 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.wal_compression: 0 2024-04-12T08:53:17.624 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.atomic_flush: 0 2024-04-12T08:53:17.624 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T08:53:17.624 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T08:53:17.624 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.log_readahead_size: 0 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_background_jobs: 2 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_background_compactions: -1 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_subcompactions: 1 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T08:53:17.625 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_open_files: -1 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_background_flushes: -1 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Compression algorithms supported: 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: kZSTD supported: 0 2024-04-12T08:53:17.626 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: kXpressCompression supported: 0 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: kZlibCompression supported: 1 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: kSnappyCompression supported: 1 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: kLZ4Compression supported: 1 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: kBZip2Compression supported: 0 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.merge_operator: 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_filter: None 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T08:53:17.627 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556ec7c0fbe0) 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: cache_index_and_filter_blocks: 1 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: pin_top_level_index_and_filter: 1 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: index_type: 0 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: data_block_index_type: 0 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: index_shortening: 1 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: checksum: 4 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: no_block_cache: 0 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: block_cache: 0x556ec7c13090 2024-04-12T08:53:17.628 INFO:journalctl@ceph.mon.b.smithi114.stdout: block_cache_name: BinnedLRUCache 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: block_cache_options: 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: capacity : 536870912 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: num_shard_bits : 4 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: strict_capacity_limit : 0 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: high_pri_pool_ratio: 0.000 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: block_cache_compressed: (nil) 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: persistent_cache: (nil) 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: block_size: 4096 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: block_size_deviation: 10 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: block_restart_interval: 16 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: index_block_restart_interval: 1 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: metadata_block_size: 4096 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: partition_filters: 0 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: use_delta_encoding: 1 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: filter_policy: bloomfilter 2024-04-12T08:53:17.629 INFO:journalctl@ceph.mon.b.smithi114.stdout: whole_key_filtering: 1 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout: verify_compression: 0 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout: read_amp_bytes_per_bit: 0 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout: format_version: 5 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout: enable_index_compression: 1 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout: block_align: 0 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout: max_auto_readahead_size: 262144 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout: prepopulate_block_cache: 0 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout: initial_auto_readahead_size: 8192 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compression: NoCompression 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T08:53:17.630 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.num_levels: 7 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T08:53:17.631 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T08:53:17.632 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T08:53:17.633 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.table_properties_collectors: 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.inplace_update_support: 0 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T08:53:17.634 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.bloom_locality: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.max_successive_merges: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.ttl: 2592000 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.enable_blob_files: false 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.min_blob_size: 0 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T08:53:17.635 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12710, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5d43af56-49a3-4866-b52b-a14e7137e0af 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911997411271, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-12T08:53:17.636 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911997436685, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 6066687, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12715, "largest_seqno": 16728, "table_properties": {"data_size": 6050881, "index_size": 10625, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4165, "raw_key_size": 35555, "raw_average_key_size": 21, "raw_value_size": 6017151, "raw_average_value_size": 3616, "num_data_blocks": 476, "num_entries": 1664, "num_filter_entries": 1664, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712911997, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5d43af56-49a3-4866-b52b-a14e7137e0af", "db_session_id": "SARUYX3UBBCTJP0Q2811", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712911997436770, "job": 1, "event": "recovery_finished"} 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556ec7d04000 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: DB pointer 0x556ec7cee000 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout: ** DB Stats ** 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T08:53:17.637 INFO:journalctl@ceph.mon.b.smithi114.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: ** Compaction Stats [default] ** 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: L0 1/0 5.79 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.6 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: L6 1/0 6.05 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: Sum 2/0 11.84 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.6 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.6 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: 2024-04-12T08:53:17.638 INFO:journalctl@ceph.mon.b.smithi114.stdout: ** Compaction Stats [default] ** 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 492.6 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: Flush(GB): cumulative 0.006, interval 0.006 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: Cumulative compaction: 0.01 GB write, 205.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: Interval compaction: 0.01 GB write, 205.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T08:53:17.639 INFO:journalctl@ceph.mon.b.smithi114.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout: Block cache BinnedLRUCache@0x556ec7c13090#2 capacity: 512.00 MB usage: 16.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout: 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: starting mon.b rank 1 at public addrs [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] at bind addrs [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: mon.b@-1(???) e3 preinit fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: mon.b@-1(???).mds e8 new map 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: mon.b@-1(???).mds e8 print_map 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout: e8 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout: legacy client fscid: 1 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout: 2024-04-12T08:53:17.640 INFO:journalctl@ceph.mon.b.smithi114.stdout: Filesystem 'foo' (1) 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: fs_name foo 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: epoch 7 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: created 2024-04-12T08:43:28.135965+0000 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: modified 2024-04-12T08:43:37.934695+0000 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: tableserver 0 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: root 0 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: session_timeout 60 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: session_autoclose 300 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: max_file_size 1099511627776 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: max_xattr_size 65536 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: required_client_features {} 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: last_failure 0 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: last_failure_osd_epoch 0 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T08:53:17.641 INFO:journalctl@ceph.mon.b.smithi114.stdout: max_mds 1 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: in 0 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: up {0=14631} 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: failed 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: damaged 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: stopped 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: data_pools [4] 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: metadata_pool 3 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: inline_data disabled 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: balancer 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: bal_rank_mask -1 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: standby_count_wanted 1 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: qdb_cluster leader: 0 members: 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: [mds.foo.smithi090.rxdrqj{0:14631} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.90:6834/3383576044,v1:172.21.15.90:6835/3383576044] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T08:53:17.642 INFO:journalctl@ceph.mon.b.smithi114.stdout: 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout: 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout: Standby daemons: 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout: 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout: [mds.foo.smithi114.nbzjob{-1:24496} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.114:6832/562045409,v1:172.21.15.114:6833/562045409] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: mon.b@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 bash[89462]: cd2de45bf1bf54bbe8467638c6a2d9ada8d431328156bed7f5547d646baa82c9 2024-04-12T08:53:17.643 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 systemd[1]: Started Ceph mon.b for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:53:18.084 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:17 smithi090 ceph-mon[99575]: pgmap v106: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2024-04-12T08:53:18.084 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:17 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:18.084 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:17 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:53:18.084 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:17 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:53:18.084 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:17 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:53:18.084 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:17 smithi090 ceph-mon[95814]: pgmap v106: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2024-04-12T08:53:18.084 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:17 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:18.084 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:17 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:53:18.084 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:17 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:53:18.084 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:17 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:53:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: pgmap v106: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2024-04-12T08:53:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:53:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:53:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:17 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:53:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:53:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: mon.b calling monitor election 2024-04-12T08:53:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: mon.c calling monitor election 2024-04-12T08:53:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: mon.a calling monitor election 2024-04-12T08:53:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:53:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:53:18.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:53:18.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: monmap epoch 4 2024-04-12T08:53:18.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:53:18.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: last_changed 2024-04-12T08:53:17.776595+0000 2024-04-12T08:53:18.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: created 2024-04-12T08:36:31.874980+0000 2024-04-12T08:53:18.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: min_mon_release 19 (squid) 2024-04-12T08:53:18.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: election_strategy: 1 2024-04-12T08:53:18.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: 0: [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon.a 2024-04-12T08:53:18.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: 1: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.b 2024-04-12T08:53:18.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: 2: [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon.c 2024-04-12T08:53:18.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:53:18.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T08:53:18.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-12T08:53:18.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:53:18.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T08:53:18.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: daemon grafana.a on smithi114 is in error state 2024-04-12T08:53:18.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='' 2024-04-12T08:53:18.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:18.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:18 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:53:19.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:53:19.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: mon.b calling monitor election 2024-04-12T08:53:19.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: mon.c calling monitor election 2024-04-12T08:53:19.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: mon.a calling monitor election 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: monmap epoch 4 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: last_changed 2024-04-12T08:53:17.776595+0000 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: created 2024-04-12T08:36:31.874980+0000 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: min_mon_release 19 (squid) 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: election_strategy: 1 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: 0: [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon.a 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: 1: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.b 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: 2: [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon.c 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T08:53:19.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: daemon grafana.a on smithi114 is in error state 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='' 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: mon.b calling monitor election 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: mon.c calling monitor election 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: mon.a calling monitor election 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T08:53:19.176 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: monmap epoch 4 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: fsid 96735376-f8a7-11ee-bc8c-c7b262605968 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: last_changed 2024-04-12T08:53:17.776595+0000 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: created 2024-04-12T08:36:31.874980+0000 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: min_mon_release 19 (squid) 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: election_strategy: 1 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: 0: [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon.a 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: 1: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.b 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: 2: [v2:172.21.15.90:3301/0,v1:172.21.15.90:6790/0] mon.c 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: fsmap foo:1 {0=foo.smithi090.rxdrqj=up:active} 1 up:standby 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T08:53:19.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: daemon grafana.a on smithi114 is in error state 2024-04-12T08:53:19.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='' 2024-04-12T08:53:19.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:19.178 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:18 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:53:19.601 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 3 2024-04-12T08:53:19.793 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'unique_pool_1' removed 2024-04-12T08:53:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:19 smithi090 ceph-mon[95814]: pgmap v108: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 504 op/s 2024-04-12T08:53:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:19 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1427900646' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:53:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:19 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/774491505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:53:19.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:19 smithi090 ceph-mon[99575]: pgmap v108: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 504 op/s 2024-04-12T08:53:19.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:19 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1427900646' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:53:19.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:19 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/774491505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:53:20.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:19 smithi114 ceph-mon[89482]: pgmap v108: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 504 op/s 2024-04-12T08:53:20.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:19 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1427900646' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:53:20.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:19 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/774491505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T08:53:20.638 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-12T08:53:20.639 INFO:tasks.radosbench:Beginning radosbench... 2024-04-12T08:53:20.639 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-04-12T08:53:20.639 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-04-12T08:53:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:20 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1427900646' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T08:53:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:20 smithi090 ceph-mon[95814]: osdmap e84: 8 total, 8 up, 8 in 2024-04-12T08:53:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:20 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1427900646' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T08:53:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:20 smithi090 ceph-mon[99575]: osdmap e84: 8 total, 8 up, 8 in 2024-04-12T08:53:21.084 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch ps' 2024-04-12T08:53:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:20 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1427900646' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T08:53:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:20 smithi114 ceph-mon[89482]: osdmap e84: 8 total, 8 up, 8 in 2024-04-12T08:53:21.185 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:53:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:21 smithi090 ceph-mon[95814]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:21 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:21 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:21 smithi090 ceph-mon[99575]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:21 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:21.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:21 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:21.948 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:53:22.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:21 smithi114 ceph-mon[89482]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:22.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:21 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:22.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:21 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:22.664 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:22 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:53:22.444+0000 7efe43a91700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-12T08:53:23.667 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:53:23] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.43.0" 2024-04-12T08:53:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:23 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:23 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:23 smithi090 ceph-mon[99575]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:23 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:23 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:23 smithi090 ceph-mon[95814]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:23 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:23 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:23 smithi114 ceph-mon[89482]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:24.639 INFO:teuthology.orchestra.run.smithi090.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T08:53:24.639 INFO:teuthology.orchestra.run.smithi090.stdout:alertmanager.a smithi090 *:9093,9094 running (5m) 14s ago 11m 24.5M - 0.25.0 c8568f914cd2 d11ba2dc752a 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:grafana.a smithi114 *:3000 error 3s ago 10m - - 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi090.rxdrqj smithi090 running (9m) 14s ago 9m 19.7M - 17.2.7-827-gd3b5060a 21de9b7369ad f627bce7c272 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi114.nbzjob smithi114 running (9m) 3s ago 9m 14.8M - 17.2.7-827-gd3b5060a 21de9b7369ad 60cb65499580 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.x smithi114 *:8443,9283,8765 running (4m) 3s ago 15m 451M - 19.0.0-2053-gb277feb2 668895944f0e 60c0ca408f91 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.y smithi090 *:8443,9283,8765 running (4m) 14s ago 16m 501M - 19.0.0-2053-gb277feb2 668895944f0e b61a0f582c61 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:mon.a smithi090 running (3m) 14s ago 16m 43.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e ad8458aaf579 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:mon.b smithi114 running (7s) 3s ago 15m 29.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e cd2de45bf1bf 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:mon.c smithi090 running (106s) 14s ago 15m 37.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e 6bfdbaae0edc 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.a smithi090 *:9100 running (5m) 14s ago 11m 17.2M - 1.5.0 0da6a335fe13 687747c6880c 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.b smithi114 *:9100 running (5m) 3s ago 11m 15.9M - 1.5.0 0da6a335fe13 e89762981b55 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:osd.0 smithi090 running (14m) 14s ago 14m 382M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 9395ed639a37 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:osd.1 smithi090 running (14m) 14s ago 14m 345M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 98eb8b5854e9 2024-04-12T08:53:24.640 INFO:teuthology.orchestra.run.smithi090.stdout:osd.2 smithi090 running (13m) 14s ago 13m 472M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 8008248b4e61 2024-04-12T08:53:24.641 INFO:teuthology.orchestra.run.smithi090.stdout:osd.3 smithi090 running (13m) 14s ago 13m 494M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 83a5641ba19a 2024-04-12T08:53:24.641 INFO:teuthology.orchestra.run.smithi090.stdout:osd.4 smithi114 running (13m) 3s ago 13m 416M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 15d0d8563392 2024-04-12T08:53:24.641 INFO:teuthology.orchestra.run.smithi090.stdout:osd.5 smithi114 running (12m) 3s ago 12m 439M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 54049e134882 2024-04-12T08:53:24.641 INFO:teuthology.orchestra.run.smithi090.stdout:osd.6 smithi114 running (12m) 3s ago 12m 312M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 86dfedc8629b 2024-04-12T08:53:24.641 INFO:teuthology.orchestra.run.smithi090.stdout:osd.7 smithi114 running (11m) 3s ago 11m 389M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad dda597b19712 2024-04-12T08:53:24.641 INFO:teuthology.orchestra.run.smithi090.stdout:prometheus.a smithi114 *:9095 running (4m) 3s ago 11m 61.5M - 2.43.0 a07b618ecd1d f009118ca428 2024-04-12T08:53:24.687 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'unique_pool_2' created 2024-04-12T08:53:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:24 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/448579041' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-12T08:53:24.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:24 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/448579041' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-12T08:53:25.007 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:24 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/448579041' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-12T08:53:25.343 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade pause' 2024-04-12T08:53:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:25 smithi114 ceph-mon[89482]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:53:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/448579041' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-12T08:53:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:25 smithi114 ceph-mon[89482]: osdmap e85: 8 total, 8 up, 8 in 2024-04-12T08:53:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:25 smithi114 ceph-mon[89482]: pgmap v113: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:25.920 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:25 smithi090 ceph-mon[99575]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:53:25.920 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/448579041' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-12T08:53:25.921 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:25 smithi090 ceph-mon[99575]: osdmap e85: 8 total, 8 up, 8 in 2024-04-12T08:53:25.921 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:25 smithi090 ceph-mon[99575]: pgmap v113: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:25.921 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:25 smithi090 ceph-mon[95814]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:53:25.921 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/448579041' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-12T08:53:25.921 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:25 smithi090 ceph-mon[95814]: osdmap e85: 8 total, 8 up, 8 in 2024-04-12T08:53:25.921 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:25 smithi090 ceph-mon[95814]: pgmap v113: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:25.943 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-04-12T08:53:26.037 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:53:26.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:53:26.692 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: osdmap e86: 8 total, 8 up, 8 in 2024-04-12T08:53:27.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:53:27.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: osdmap e86: 8 total, 8 up, 8 in 2024-04-12T08:53:27.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:27.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:27.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:27.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:53:27.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: pgmap v115: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:27.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: pgmap v116: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:27.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:27.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: Reconfiguring mon.a (monmap changed)... 2024-04-12T08:53:27.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:27.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:27.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:27.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:26 smithi114 ceph-mon[89482]: Reconfiguring daemon mon.a on smithi090 2024-04-12T08:53:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:53:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: pgmap v115: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: pgmap v116: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: Reconfiguring mon.a (monmap changed)... 2024-04-12T08:53:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[95814]: Reconfiguring daemon mon.a on smithi090 2024-04-12T08:53:27.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:53:27.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: osdmap e86: 8 total, 8 up, 8 in 2024-04-12T08:53:27.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: pgmap v115: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: pgmap v116: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: Reconfiguring mon.a (monmap changed)... 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:26 smithi090 ceph-mon[99575]: Reconfiguring daemon mon.a on smithi090 2024-04-12T08:53:27.258 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:53:27.830 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:27 smithi090 ceph-mon[99575]: osdmap e87: 8 total, 8 up, 8 in 2024-04-12T08:53:27.831 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:27 smithi090 ceph-mon[95814]: osdmap e87: 8 total, 8 up, 8 in 2024-04-12T08:53:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:27 smithi114 ceph-mon[89482]: osdmap e87: 8 total, 8 up, 8 in 2024-04-12T08:53:28.890 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:28 smithi090 ceph-mon[99575]: pgmap v118: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:28.891 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:28 smithi090 ceph-mon[95814]: pgmap v118: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:29.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:28 smithi114 ceph-mon[89482]: pgmap v118: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:29.816 INFO:teuthology.orchestra.run.smithi090.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:53:30.150 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:30.150 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[95814]: Reconfiguring mgr.y (monmap changed)... 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[95814]: Reconfiguring daemon mgr.y on smithi090 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[99575]: Reconfiguring mgr.y (monmap changed)... 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:30.151 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:30.152 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:53:30.152 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:30.152 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[99575]: Reconfiguring daemon mgr.y on smithi090 2024-04-12T08:53:30.152 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:29 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:30.238 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:29 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:30.238 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:29 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:30.238 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:29 smithi114 ceph-mon[89482]: Reconfiguring mgr.y (monmap changed)... 2024-04-12T08:53:30.238 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:29 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:30.238 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:29 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:30.238 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:29 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:53:30.238 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:29 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:30.239 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:29 smithi114 ceph-mon[89482]: Reconfiguring daemon mgr.y on smithi090 2024-04-12T08:53:30.239 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:29 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:31.767 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:31 smithi090 ceph-mon[99575]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:53:31.767 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:31 smithi090 ceph-mon[99575]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:53:31.768 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:31 smithi090 ceph-mon[99575]: pgmap v119: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:31.768 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:31 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:31.768 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:31.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:31 smithi090 ceph-mon[95814]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:53:31.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:31 smithi090 ceph-mon[95814]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:53:31.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:31 smithi090 ceph-mon[95814]: pgmap v119: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:31.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:31 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:31.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:31 smithi114 ceph-mon[89482]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:53:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:31 smithi114 ceph-mon[89482]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:53:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:31 smithi114 ceph-mon[89482]: pgmap v119: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:31 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:31.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:32.608 INFO:teuthology.orchestra.run.smithi090.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-04-12T08:53:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[99575]: Reconfiguring mon.c (monmap changed)... 2024-04-12T08:53:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[99575]: Reconfiguring daemon mon.c on smithi090 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[99575]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2150099204' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[99575]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[95814]: Reconfiguring mon.c (monmap changed)... 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:32.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[95814]: Reconfiguring daemon mon.c on smithi090 2024-04-12T08:53:32.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[95814]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:32.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2150099204' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:53:32.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:32 smithi090 ceph-mon[95814]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:53:32.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:32 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:32.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:32 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:32.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:32 smithi114 ceph-mon[89482]: Reconfiguring mon.c (monmap changed)... 2024-04-12T08:53:32.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:32 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:32.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:32 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:32.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:32 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:32.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:32 smithi114 ceph-mon[89482]: Reconfiguring daemon mon.c on smithi090 2024-04-12T08:53:32.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:32 smithi114 ceph-mon[89482]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:32.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:32 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2150099204' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:53:32.870 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:32 smithi114 ceph-mon[89482]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:53:33.045 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'sleep 60' 2024-04-12T08:53:33.388 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:53:33] "GET /metrics HTTP/1.1" 200 36277 "" "Prometheus/2.43.0" 2024-04-12T08:53:33.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:33 smithi090 ceph-mon[95814]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:53:33.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:33 smithi090 ceph-mon[95814]: osdmap e88: 8 total, 8 up, 8 in 2024-04-12T08:53:33.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:33 smithi090 ceph-mon[99575]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:53:33.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:33 smithi090 ceph-mon[99575]: osdmap e88: 8 total, 8 up, 8 in 2024-04-12T08:53:33.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:33 smithi114 ceph-mon[89482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:53:33.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:33 smithi114 ceph-mon[89482]: osdmap e88: 8 total, 8 up, 8 in 2024-04-12T08:53:34.599 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:53:35.094 DEBUG:teuthology.orchestra.run.smithi090:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-04-12T08:53:35.097 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-12T08:53:35.097 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-12T08:53:35.141 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:hints = 1 2024-04-12T08:53:35.141 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-12T08:53:35.141 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Object prefix: benchmark_data_smithi090_105619 2024-04-12T08:53:35.141 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:53:35.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 0 0 0 0 0 0 - 0 2024-04-12T08:53:35.777 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[99575]: pgmap v122: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:35.777 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:35.777 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:35.777 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T08:53:35.777 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:35.777 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[99575]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:53:35.777 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[95814]: pgmap v122: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:35.777 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:35.777 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:35.777 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T08:53:35.778 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:35.778 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:35 smithi090 ceph-mon[95814]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:53:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:35 smithi114 ceph-mon[89482]: pgmap v122: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:35 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:35 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:35.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:35 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T08:53:35.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:35 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:35.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:35 smithi114 ceph-mon[89482]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:53:36.141 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 1 16 3422 3406 212.863 212.875 0.00131087 0.00429524 2024-04-12T08:53:36.574 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:36 smithi090 ceph-mon[95814]: Reconfiguring osd.0 (monmap changed)... 2024-04-12T08:53:36.574 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:36 smithi090 ceph-mon[95814]: Reconfiguring daemon osd.0 on smithi090 2024-04-12T08:53:36.574 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:36 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:36.574 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:36 smithi090 ceph-mon[99575]: Reconfiguring osd.0 (monmap changed)... 2024-04-12T08:53:36.574 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:36 smithi090 ceph-mon[99575]: Reconfiguring daemon osd.0 on smithi090 2024-04-12T08:53:36.574 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:36 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:36 smithi114 ceph-mon[89482]: Reconfiguring osd.0 (monmap changed)... 2024-04-12T08:53:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:36 smithi114 ceph-mon[89482]: Reconfiguring daemon osd.0 on smithi090 2024-04-12T08:53:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:36 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:37.141 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 2 16 6880 6864 214.482 216.125 0.00226569 0.00465352 2024-04-12T08:53:37.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:37 smithi090 ceph-mon[99575]: pgmap v123: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:37.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:37 smithi090 ceph-mon[95814]: pgmap v123: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:37.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:37 smithi114 ceph-mon[89482]: pgmap v123: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-12T08:53:38.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 3 16 10254 10238 213.273 210.875 0.0047794 0.00468045 2024-04-12T08:53:39.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 4 16 13116 13100 204.67 178.875 0.00260126 0.00487969 2024-04-12T08:53:39.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:39.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:39.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[95814]: Reconfiguring osd.1 (monmap changed)... 2024-04-12T08:53:39.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T08:53:39.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:39.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[95814]: Reconfiguring daemon osd.1 on smithi090 2024-04-12T08:53:39.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[95814]: pgmap v124: 89 pgs: 89 active+clean; 322 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s wr, 514 op/s 2024-04-12T08:53:39.293 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:39.293 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:39.294 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[99575]: Reconfiguring osd.1 (monmap changed)... 2024-04-12T08:53:39.294 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T08:53:39.294 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:39.294 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[99575]: Reconfiguring daemon osd.1 on smithi090 2024-04-12T08:53:39.294 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:39 smithi090 ceph-mon[99575]: pgmap v124: 89 pgs: 89 active+clean; 322 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s wr, 514 op/s 2024-04-12T08:53:39.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:39 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:39.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:39 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:39.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:39 smithi114 ceph-mon[89482]: Reconfiguring osd.1 (monmap changed)... 2024-04-12T08:53:39.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:39 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T08:53:39.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:39 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:39.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:39 smithi114 ceph-mon[89482]: Reconfiguring daemon osd.1 on smithi090 2024-04-12T08:53:39.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:39 smithi114 ceph-mon[89482]: pgmap v124: 89 pgs: 89 active+clean; 322 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s wr, 514 op/s 2024-04-12T08:53:40.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 5 16 16005 15989 199.845 180.562 0.00527158 0.00499721 2024-04-12T08:53:41.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 6 16 18688 18672 194.483 167.688 0.00897785 0.00513622 2024-04-12T08:53:41.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:41 smithi114 ceph-mon[89482]: pgmap v125: 89 pgs: 89 active+clean; 322 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s wr, 514 op/s 2024-04-12T08:53:41.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:41 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:41.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:41 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:41.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:41 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T08:53:41.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:41 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:41 smithi090 ceph-mon[95814]: pgmap v125: 89 pgs: 89 active+clean; 322 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s wr, 514 op/s 2024-04-12T08:53:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:41 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:41 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:41 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T08:53:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:41 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:41.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:41 smithi090 ceph-mon[99575]: pgmap v125: 89 pgs: 89 active+clean; 322 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s wr, 514 op/s 2024-04-12T08:53:41.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:41 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:41.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:41 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:41.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:41 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T08:53:41.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:41 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:42.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 7 16 21173 21157 188.885 155.312 0.00707028 0.00528861 2024-04-12T08:53:42.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:42 smithi114 ceph-mon[89482]: Reconfiguring osd.2 (monmap changed)... 2024-04-12T08:53:42.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:42 smithi114 ceph-mon[89482]: Reconfiguring daemon osd.2 on smithi090 2024-04-12T08:53:42.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:42 smithi090 ceph-mon[99575]: Reconfiguring osd.2 (monmap changed)... 2024-04-12T08:53:42.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:42 smithi090 ceph-mon[99575]: Reconfiguring daemon osd.2 on smithi090 2024-04-12T08:53:42.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:42 smithi090 ceph-mon[95814]: Reconfiguring osd.2 (monmap changed)... 2024-04-12T08:53:42.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:42 smithi090 ceph-mon[95814]: Reconfiguring daemon osd.2 on smithi090 2024-04-12T08:53:43.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 8 16 23452 23436 183.078 142.438 0.00667591 0.00545677 2024-04-12T08:53:43.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:53:43] "GET /metrics HTTP/1.1" 200 36277 "" "Prometheus/2.43.0" 2024-04-12T08:53:43.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:43 smithi090 ceph-mon[99575]: pgmap v126: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-04-12T08:53:43.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:43 smithi090 ceph-mon[95814]: pgmap v126: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-04-12T08:53:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:43 smithi114 ceph-mon[89482]: pgmap v126: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-04-12T08:53:44.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 9 16 25788 25772 178.957 146 0.00817457 0.0055816 2024-04-12T08:53:45.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 10 16 28112 28096 175.585 145.25 0.0136058 0.00568955 2024-04-12T08:53:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:45 smithi114 ceph-mon[89482]: pgmap v127: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-04-12T08:53:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T08:53:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:45 smithi090 ceph-mon[95814]: pgmap v127: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-04-12T08:53:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T08:53:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:45.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:45 smithi090 ceph-mon[99575]: pgmap v127: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-04-12T08:53:45.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:45.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:45.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T08:53:45.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:46.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 11 16 30212 30196 171.553 131.25 0.00427651 0.00582495 2024-04-12T08:53:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:46 smithi090 ceph-mon[99575]: Reconfiguring osd.3 (monmap changed)... 2024-04-12T08:53:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:46 smithi090 ceph-mon[99575]: Reconfiguring daemon osd.3 on smithi090 2024-04-12T08:53:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:46 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:46 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:46.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:46 smithi090 ceph-mon[95814]: Reconfiguring osd.3 (monmap changed)... 2024-04-12T08:53:46.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:46 smithi090 ceph-mon[95814]: Reconfiguring daemon osd.3 on smithi090 2024-04-12T08:53:46.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:46 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:46.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:46 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:46 smithi114 ceph-mon[89482]: Reconfiguring osd.3 (monmap changed)... 2024-04-12T08:53:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:46 smithi114 ceph-mon[89482]: Reconfiguring daemon osd.3 on smithi090 2024-04-12T08:53:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:46 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:46.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:46 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:53:47.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 12 16 32094 32078 167.058 117.625 0.00185508 0.00594573 2024-04-12T08:53:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:47 smithi114 ceph-mon[89482]: pgmap v128: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-04-12T08:53:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:47 smithi090 ceph-mon[99575]: pgmap v128: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-04-12T08:53:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:47 smithi090 ceph-mon[95814]: pgmap v128: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-04-12T08:53:48.142 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 13 16 33703 33687 161.942 100.562 0.00482178 0.00616939 2024-04-12T08:53:49.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 14 16 35305 35289 157.526 100.125 0.00484876 0.00634275 2024-04-12T08:53:49.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:49 smithi114 ceph-mon[89482]: pgmap v129: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 163 MiB/s wr, 2.60k op/s 2024-04-12T08:53:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:49 smithi090 ceph-mon[99575]: pgmap v129: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 163 MiB/s wr, 2.60k op/s 2024-04-12T08:53:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:49 smithi090 ceph-mon[95814]: pgmap v129: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 163 MiB/s wr, 2.60k op/s 2024-04-12T08:53:50.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 15 16 36425 36409 151.69 70 0.138476 0.00652107 2024-04-12T08:53:51.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 16 16 37811 37795 147.623 86.625 0.00681882 0.00676865 2024-04-12T08:53:51.774 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:51 smithi090 ceph-mon[99575]: pgmap v130: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-04-12T08:53:51.774 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:51 smithi090 ceph-mon[95814]: pgmap v130: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-04-12T08:53:51.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:51 smithi114 ceph-mon[89482]: pgmap v130: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-04-12T08:53:52.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 17 16 39129 39113 143.784 82.375 0.00296289 0.00688879 2024-04-12T08:53:53.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 18 16 40799 40783 141.594 104.375 0.00673252 0.0070559 2024-04-12T08:53:53.410 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:53:53] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.43.0" 2024-04-12T08:53:53.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[95814]: pgmap v131: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 174 MiB/s wr, 2.79k op/s 2024-04-12T08:53:53.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:53.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:53.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:53:53.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:53:53.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:53.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[99575]: pgmap v131: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 174 MiB/s wr, 2.79k op/s 2024-04-12T08:53:53.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:53.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:53.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:53:53.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:53:53.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:53 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:53 smithi114 ceph-mon[89482]: pgmap v131: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 174 MiB/s wr, 2.79k op/s 2024-04-12T08:53:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:53 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:53 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:53 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:53:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:53 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi090.rxdrqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:53:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:53 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:54.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 19 16 42356 42340 139.263 97.3125 0.0110486 0.00717536 2024-04-12T08:53:54.839 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:54 smithi090 ceph-mon[95814]: Reconfiguring mds.foo.smithi090.rxdrqj (monmap changed)... 2024-04-12T08:53:54.839 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:54 smithi090 ceph-mon[95814]: Reconfiguring daemon mds.foo.smithi090.rxdrqj on smithi090 2024-04-12T08:53:54.839 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:54 smithi090 ceph-mon[99575]: Reconfiguring mds.foo.smithi090.rxdrqj (monmap changed)... 2024-04-12T08:53:54.839 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:54 smithi090 ceph-mon[99575]: Reconfiguring daemon mds.foo.smithi090.rxdrqj on smithi090 2024-04-12T08:53:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:54 smithi114 ceph-mon[89482]: Reconfiguring mds.foo.smithi090.rxdrqj (monmap changed)... 2024-04-12T08:53:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:54 smithi114 ceph-mon[89482]: Reconfiguring daemon mds.foo.smithi090.rxdrqj on smithi090 2024-04-12T08:53:55.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:53:55.143818+0000 min lat: 0.000947928 max lat: 0.224002 avg lat: 0.00726432 2024-04-12T08:53:55.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:53:55.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 20 16 44035 44019 137.546 104.938 0.0081331 0.00726432 2024-04-12T08:53:55.815 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:55 smithi114 ceph-mon[89482]: pgmap v132: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-04-12T08:53:55.815 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:55 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:55.815 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:55 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:55.815 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:55 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:55.815 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:55 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:55.815 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:55 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[95814]: pgmap v132: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-04-12T08:53:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[99575]: pgmap v132: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-04-12T08:53:55.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:55.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:55.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T08:53:55.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T08:53:55.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:55 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:56.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 21 16 45907 45891 136.567 117 0.00907478 0.00731676 2024-04-12T08:53:56.859 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:56 smithi114 ceph-mon[89482]: Reconfiguring mon.b (monmap changed)... 2024-04-12T08:53:56.859 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:56 smithi114 ceph-mon[89482]: Reconfiguring daemon mon.b on smithi114 2024-04-12T08:53:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:56 smithi090 ceph-mon[95814]: Reconfiguring mon.b (monmap changed)... 2024-04-12T08:53:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:56 smithi090 ceph-mon[95814]: Reconfiguring daemon mon.b on smithi114 2024-04-12T08:53:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:56 smithi090 ceph-mon[99575]: Reconfiguring mon.b (monmap changed)... 2024-04-12T08:53:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:56 smithi090 ceph-mon[99575]: Reconfiguring daemon mon.b on smithi114 2024-04-12T08:53:57.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 22 16 47723 47707 135.518 113.5 0.00499104 0.007374 2024-04-12T08:53:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:57 smithi114 ceph-mon[89482]: pgmap v133: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-04-12T08:53:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:57 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:57 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:57.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:57 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:57.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:57 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:57.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:57 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:53:57.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:57 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[95814]: pgmap v133: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-04-12T08:53:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:53:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:57.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[99575]: pgmap v133: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-04-12T08:53:57.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:57.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:57.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:57.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T08:53:57.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T08:53:57.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:57 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:58.143 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 23 16 49378 49362 134.123 103.438 0.00634988 0.00745089 2024-04-12T08:53:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:58 smithi114 ceph-mon[89482]: Reconfiguring mgr.x (monmap changed)... 2024-04-12T08:53:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:58 smithi114 ceph-mon[89482]: Reconfiguring daemon mgr.x on smithi114 2024-04-12T08:53:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:58 smithi090 ceph-mon[95814]: Reconfiguring mgr.x (monmap changed)... 2024-04-12T08:53:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:58 smithi090 ceph-mon[95814]: Reconfiguring daemon mgr.x on smithi114 2024-04-12T08:53:58.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:58 smithi090 ceph-mon[99575]: Reconfiguring mgr.x (monmap changed)... 2024-04-12T08:53:58.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:58 smithi090 ceph-mon[99575]: Reconfiguring daemon mgr.x on smithi114 2024-04-12T08:53:59.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 24 16 51127 51111 133.089 109.312 0.00976964 0.00750813 2024-04-12T08:53:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:59 smithi114 ceph-mon[89482]: pgmap v134: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-12T08:53:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:59 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:59 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:59 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T08:53:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:53:59 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:59.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:59 smithi090 ceph-mon[95814]: pgmap v134: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-12T08:53:59.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:59 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:59.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:59 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:59.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:59 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T08:53:59.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:53:59 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:53:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:59 smithi090 ceph-mon[99575]: pgmap v134: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-12T08:53:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:59 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:59 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:53:59.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:59 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T08:53:59.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:53:59 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:00.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 25 16 52837 52821 132.04 106.875 0.00784528 0.00756795 2024-04-12T08:54:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:00 smithi114 ceph-mon[89482]: Reconfiguring osd.4 (monmap changed)... 2024-04-12T08:54:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:00 smithi114 ceph-mon[89482]: Reconfiguring daemon osd.4 on smithi114 2024-04-12T08:54:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:00 smithi090 ceph-mon[95814]: Reconfiguring osd.4 (monmap changed)... 2024-04-12T08:54:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:00 smithi090 ceph-mon[95814]: Reconfiguring daemon osd.4 on smithi114 2024-04-12T08:54:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:00 smithi090 ceph-mon[99575]: Reconfiguring osd.4 (monmap changed)... 2024-04-12T08:54:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:00 smithi090 ceph-mon[99575]: Reconfiguring daemon osd.4 on smithi114 2024-04-12T08:54:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:01.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 26 16 54447 54431 130.832 100.625 0.00981816 0.00763766 2024-04-12T08:54:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:01 smithi114 ceph-mon[89482]: pgmap v135: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T08:54:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:01 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T08:54:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:01 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:01.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:01 smithi090 ceph-mon[95814]: pgmap v135: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T08:54:01.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:01.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:01.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T08:54:01.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:01.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:01 smithi090 ceph-mon[99575]: pgmap v135: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T08:54:01.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:01.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:01.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T08:54:01.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:02.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 27 16 56084 56068 129.775 102.312 0.00764983 0.00769991 2024-04-12T08:54:02.864 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:02 smithi114 ceph-mon[89482]: Reconfiguring osd.5 (monmap changed)... 2024-04-12T08:54:02.865 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:02 smithi114 ceph-mon[89482]: Reconfiguring daemon osd.5 on smithi114 2024-04-12T08:54:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:02 smithi090 ceph-mon[95814]: Reconfiguring osd.5 (monmap changed)... 2024-04-12T08:54:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:02 smithi090 ceph-mon[95814]: Reconfiguring daemon osd.5 on smithi114 2024-04-12T08:54:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:02 smithi090 ceph-mon[99575]: Reconfiguring osd.5 (monmap changed)... 2024-04-12T08:54:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:02 smithi090 ceph-mon[99575]: Reconfiguring daemon osd.5 on smithi114 2024-04-12T08:54:03.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 28 16 57684 57668 128.711 100 0.0079227 0.00776441 2024-04-12T08:54:03.562 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:54:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:54:03] "GET /metrics HTTP/1.1" 200 36271 "" "Prometheus/2.43.0" 2024-04-12T08:54:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:03 smithi114 ceph-mon[89482]: pgmap v136: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 125 MiB/s wr, 2.01k op/s 2024-04-12T08:54:03.873 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:03 smithi090 ceph-mon[95814]: pgmap v136: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 125 MiB/s wr, 2.01k op/s 2024-04-12T08:54:03.873 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:03 smithi090 ceph-mon[99575]: pgmap v136: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 125 MiB/s wr, 2.01k op/s 2024-04-12T08:54:04.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 29 16 59185 59169 127.508 93.8125 0.0107142 0.00783622 2024-04-12T08:54:04.840 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:04 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:04.840 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:04 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:04.840 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:04 smithi114 ceph-mon[89482]: Reconfiguring osd.6 (monmap changed)... 2024-04-12T08:54:04.840 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:04 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T08:54:04.840 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:04 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:04.841 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:04 smithi114 ceph-mon[89482]: Reconfiguring daemon osd.6 on smithi114 2024-04-12T08:54:04.841 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:04 smithi114 ceph-mon[89482]: pgmap v137: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T08:54:05.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 30 16 60684 60668 126.38 93.6875 0.00678175 0.00790773 2024-04-12T08:54:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[95814]: Reconfiguring osd.6 (monmap changed)... 2024-04-12T08:54:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T08:54:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[95814]: Reconfiguring daemon osd.6 on smithi114 2024-04-12T08:54:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[95814]: pgmap v137: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T08:54:05.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:05.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:05.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[99575]: Reconfiguring osd.6 (monmap changed)... 2024-04-12T08:54:05.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T08:54:05.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:05.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[99575]: Reconfiguring daemon osd.6 on smithi114 2024-04-12T08:54:05.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:04 smithi090 ceph-mon[99575]: pgmap v137: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T08:54:06.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 31 16 62207 62191 125.373 95.1875 0.00787501 0.00797097 2024-04-12T08:54:06.967 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:06 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:06.967 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:06 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:06.968 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:06 smithi114 ceph-mon[89482]: Reconfiguring osd.7 (monmap changed)... 2024-04-12T08:54:06.968 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:06 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T08:54:06.968 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:06 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:06.968 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:06 smithi114 ceph-mon[89482]: Reconfiguring daemon osd.7 on smithi114 2024-04-12T08:54:06.968 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:06 smithi114 ceph-mon[89482]: pgmap v138: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T08:54:07.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 32 16 63711 63695 124.393 94 0.00733243 0.00803418 2024-04-12T08:54:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[95814]: Reconfiguring osd.7 (monmap changed)... 2024-04-12T08:54:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T08:54:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[95814]: Reconfiguring daemon osd.7 on smithi114 2024-04-12T08:54:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[95814]: pgmap v138: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T08:54:07.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:07.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:07.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[99575]: Reconfiguring osd.7 (monmap changed)... 2024-04-12T08:54:07.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T08:54:07.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:07.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[99575]: Reconfiguring daemon osd.7 on smithi114 2024-04-12T08:54:07.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:06 smithi090 ceph-mon[99575]: pgmap v138: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T08:54:08.144 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 33 16 65183 65167 123.411 92 0.0066778 0.00809894 2024-04-12T08:54:09.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 34 16 66560 66544 122.312 86.0625 0.00423677 0.00816985 2024-04-12T08:54:09.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:09 smithi114 ceph-mon[89482]: pgmap v139: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-04-12T08:54:09.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:09 smithi090 ceph-mon[95814]: pgmap v139: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-04-12T08:54:09.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:09 smithi090 ceph-mon[99575]: pgmap v139: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-04-12T08:54:10.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 35 16 67819 67803 121.066 78.6875 0.0508801 0.00825291 2024-04-12T08:54:11.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 36 16 69237 69221 120.164 88.625 0.00737757 0.00831726 2024-04-12T08:54:11.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:11 smithi114 ceph-mon[89482]: pgmap v140: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:54:11.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:11 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:11.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:11 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:11.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:11 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:54:11.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:11 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:54:11.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:11 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[99575]: pgmap v140: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:54:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:54:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:54:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:11.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[95814]: pgmap v140: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:54:11.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:11.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:11.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:54:11.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi114.nbzjob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T08:54:11.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:12.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 37 16 70620 70604 119.252 86.4375 0.00942884 0.00838034 2024-04-12T08:54:12.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:12 smithi114 ceph-mon[89482]: Reconfiguring mds.foo.smithi114.nbzjob (monmap changed)... 2024-04-12T08:54:12.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:12 smithi114 ceph-mon[89482]: Reconfiguring daemon mds.foo.smithi114.nbzjob on smithi114 2024-04-12T08:54:12.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:12 smithi090 ceph-mon[95814]: Reconfiguring mds.foo.smithi114.nbzjob (monmap changed)... 2024-04-12T08:54:12.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:12 smithi090 ceph-mon[95814]: Reconfiguring daemon mds.foo.smithi114.nbzjob on smithi114 2024-04-12T08:54:12.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:12 smithi090 ceph-mon[99575]: Reconfiguring mds.foo.smithi114.nbzjob (monmap changed)... 2024-04-12T08:54:12.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:12 smithi090 ceph-mon[99575]: Reconfiguring daemon mds.foo.smithi114.nbzjob on smithi114 2024-04-12T08:54:13.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 38 16 71917 71901 118.247 81.0625 0.0130205 0.00845188 2024-04-12T08:54:13.675 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:54:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:54:13] "GET /metrics HTTP/1.1" 200 36271 "" "Prometheus/2.43.0" 2024-04-12T08:54:13.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:13 smithi090 ceph-mon[95814]: pgmap v141: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-04-12T08:54:13.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:13.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:13.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:13 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:54:13.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:13 smithi090 ceph-mon[99575]: pgmap v141: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-04-12T08:54:13.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:13 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:13.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:13 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:13.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:13 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:54:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:13 smithi114 ceph-mon[89482]: pgmap v141: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-04-12T08:54:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:13 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:13 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:13 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:54:14.145 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 39 16 73105 73089 117.118 74.25 0.00675202 0.00853265 2024-04-12T08:54:15.146 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:54:15.146111+0000 min lat: 0.000947928 max lat: 0.224002 avg lat: 0.00860817 2024-04-12T08:54:15.146 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:54:15.146 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 40 16 74332 74316 116.107 76.6875 0.0105583 0.00860817 2024-04-12T08:54:15.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:15 smithi090 ceph-mon[99575]: pgmap v142: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-12T08:54:15.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:15 smithi090 ceph-mon[95814]: pgmap v142: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-12T08:54:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:15 smithi114 ceph-mon[89482]: pgmap v142: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-12T08:54:16.146 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 41 16 75518 75502 115.082 74.125 0.0113114 0.00868452 2024-04-12T08:54:16.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:16 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:16.699 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:16.768 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:16 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:17.146 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 42 16 76756 76740 114.183 77.375 0.00900901 0.00875242 2024-04-12T08:54:17.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:17 smithi090 ceph-mon[99575]: pgmap v143: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-12T08:54:17.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:17 smithi090 ceph-mon[95814]: pgmap v143: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-12T08:54:17.779 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:17 smithi114 ceph-mon[89482]: pgmap v143: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-12T08:54:18.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 43 16 77848 77832 113.114 68.25 0.011548 0.00883572 2024-04-12T08:54:19.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 44 16 78936 78920 112.089 68 0.00993094 0.00891705 2024-04-12T08:54:19.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:19 smithi114 ceph-mon[89482]: pgmap v144: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-04-12T08:54:19.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:19 smithi090 ceph-mon[95814]: pgmap v144: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-04-12T08:54:19.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:19 smithi090 ceph-mon[99575]: pgmap v144: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-04-12T08:54:20.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 45 16 79856 79840 110.876 57.5 0.0127681 0.00901294 2024-04-12T08:54:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:20 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:20 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:20 smithi114 ceph-mon[89482]: pgmap v145: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:54:21.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 46 16 80995 80979 110.013 71.1875 0.0112812 0.00908414 2024-04-12T08:54:21.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:20 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:21.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:20 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:21.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:20 smithi090 ceph-mon[95814]: pgmap v145: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:54:21.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:20 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:21.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:20 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:21.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:20 smithi090 ceph-mon[99575]: pgmap v145: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:54:22.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:21 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:22.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:21 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:54:22.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:21 smithi114 ceph-mon[89482]: pgmap v146: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-12T08:54:22.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:21 smithi114 ceph-mon[89482]: pgmap v147: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s 2024-04-12T08:54:22.119 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:21 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:22.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 47 16 82229 82213 109.313 77.125 0.010936 0.00914326 2024-04-12T08:54:22.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:21 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:22.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:21 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:54:22.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:21 smithi090 ceph-mon[95814]: pgmap v146: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-12T08:54:22.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:21 smithi090 ceph-mon[95814]: pgmap v147: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s 2024-04-12T08:54:22.176 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:21 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:22.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:21 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:22.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:21 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:54:22.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:21 smithi090 ceph-mon[99575]: pgmap v146: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-12T08:54:22.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:21 smithi090 ceph-mon[99575]: pgmap v147: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s 2024-04-12T08:54:22.177 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:21 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:23.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 48 16 83291 83275 108.418 66.375 0.0151457 0.00921838 2024-04-12T08:54:23.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:54:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:54:23] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.43.0" 2024-04-12T08:54:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:23 smithi114 ceph-mon[89482]: pgmap v148: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T08:54:24.123 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:23 smithi090 ceph-mon[95814]: pgmap v148: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T08:54:24.123 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:23 smithi090 ceph-mon[99575]: pgmap v148: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T08:54:24.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 49 16 84338 84322 107.541 65.4375 0.00740447 0.00929404 2024-04-12T08:54:25.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 50 16 85540 85524 106.893 75.125 0.0128937 0.00935021 2024-04-12T08:54:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:25 smithi114 ceph-mon[89482]: pgmap v149: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T08:54:26.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 51 16 86711 86695 106.232 73.1875 0.0123325 0.00940853 2024-04-12T08:54:26.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:25 smithi090 ceph-mon[99575]: pgmap v149: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T08:54:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:25 smithi090 ceph-mon[95814]: pgmap v149: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T08:54:27.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 52 16 87900 87884 105.618 74.3125 0.00623351 0.00946314 2024-04-12T08:54:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:27 smithi114 ceph-mon[89482]: pgmap v150: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:54:28.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 53 16 89070 89054 105.004 73.125 0.00482567 0.00951818 2024-04-12T08:54:28.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:27 smithi090 ceph-mon[99575]: pgmap v150: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:54:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:27 smithi090 ceph-mon[95814]: pgmap v150: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T08:54:29.147 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 54 16 90225 90209 104.397 72.1875 0.00817448 0.0095735 2024-04-12T08:54:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:29 smithi114 ceph-mon[89482]: pgmap v151: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-12T08:54:30.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 55 16 91371 91355 103.801 71.625 0.00647537 0.00962859 2024-04-12T08:54:30.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:29 smithi090 ceph-mon[99575]: pgmap v151: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-12T08:54:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:29 smithi090 ceph-mon[95814]: pgmap v151: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-12T08:54:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:30 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:31.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 56 16 92515 92499 103.224 71.5 0.00820908 0.00968259 2024-04-12T08:54:31.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:30 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:31.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:30 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:32.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:31 smithi114 ceph-mon[89482]: pgmap v152: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-12T08:54:32.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 57 16 93653 93637 102.661 71.125 0.00748655 0.00973578 2024-04-12T08:54:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:31 smithi090 ceph-mon[99575]: pgmap v152: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-12T08:54:32.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:31 smithi090 ceph-mon[95814]: pgmap v152: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-12T08:54:33.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 58 16 94769 94753 102.093 69.75 0.0174265 0.00978996 2024-04-12T08:54:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:54:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:54:33] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.43.0" 2024-04-12T08:54:34.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:33 smithi114 ceph-mon[89482]: pgmap v153: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-12T08:54:34.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 59 16 95860 95844 101.518 68.1875 0.0108516 0.00984594 2024-04-12T08:54:34.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:33 smithi090 ceph-mon[95814]: pgmap v153: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-12T08:54:34.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:33 smithi090 ceph-mon[99575]: pgmap v153: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-12T08:54:35.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:54:35.148648+0000 min lat: 0.000947928 max lat: 0.224002 avg lat: 0.00990974 2024-04-12T08:54:35.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:54:35.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 60 16 96845 96829 100.852 61.5625 0.0153192 0.00990974 2024-04-12T08:54:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:35 smithi114 ceph-mon[89482]: pgmap v154: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2024-04-12T08:54:36.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 61 15 97945 97930 100.327 68.8125 0.00560757 0.00996217 2024-04-12T08:54:36.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:35 smithi090 ceph-mon[99575]: pgmap v154: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2024-04-12T08:54:36.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:35 smithi090 ceph-mon[95814]: pgmap v154: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2024-04-12T08:54:37.148 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 62 16 98986 98970 99.757 65 0.0242605 0.010019 2024-04-12T08:54:37.718 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade resume' 2024-04-12T08:54:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:37 smithi114 ceph-mon[89482]: pgmap v155: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-12T08:54:38.149 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 63 16 99977 99961 99.1549 61.9375 0.032933 0.0100794 2024-04-12T08:54:38.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:37 smithi090 ceph-mon[95814]: pgmap v155: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-12T08:54:38.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:37 smithi090 ceph-mon[99575]: pgmap v155: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-12T08:54:38.230 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:54:39.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 64 16 100966 100950 98.5714 61.8125 0.00811408 0.01014 2024-04-12T08:54:40.109 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:39 smithi090 ceph-mon[95814]: pgmap v156: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-12T08:54:40.109 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:39 smithi090 ceph-mon[99575]: pgmap v156: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-12T08:54:40.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:39 smithi114 ceph-mon[89482]: pgmap v156: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-12T08:54:40.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 65 16 101993 101977 98.0423 64.1875 0.00619151 0.010194 2024-04-12T08:54:41.065 INFO:teuthology.orchestra.run.smithi090.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:54:41.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 66 16 102903 102887 97.4185 56.875 0.00872118 0.0102592 2024-04-12T08:54:41.915 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'echo wait for half of osds to upgrade' 2024-04-12T08:54:42.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 67 16 103735 103719 96.7406 52 0.0562245 0.0103314 2024-04-12T08:54:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:42 smithi114 ceph-mon[89482]: pgmap v157: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-04-12T08:54:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:42 smithi114 ceph-mon[89482]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:54:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:42 smithi114 ceph-mon[89482]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:54:42.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:42 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:42.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:42 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:54:42.416 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:54:42.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:42 smithi090 ceph-mon[99575]: pgmap v157: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-04-12T08:54:42.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:42 smithi090 ceph-mon[99575]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:54:42.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:42 smithi090 ceph-mon[99575]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:54:42.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:42 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:42.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:42 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:54:42.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:42 smithi090 ceph-mon[95814]: pgmap v157: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-04-12T08:54:42.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:42 smithi090 ceph-mon[95814]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T08:54:42.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:42 smithi090 ceph-mon[95814]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:54:42.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:42 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:42.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:42 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:54:43.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 68 16 104573 104557 96.0881 52.375 0.01162 0.0104018 2024-04-12T08:54:43.611 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:54:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:54:43] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.43.0" 2024-04-12T08:54:44.116 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:44 smithi114 ceph-mon[89482]: pgmap v158: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:54:44.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 69 16 105570 105554 95.5985 62.3125 0.00887267 0.0104557 2024-04-12T08:54:44.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:44 smithi090 ceph-mon[95814]: pgmap v158: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:54:44.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:44 smithi090 ceph-mon[99575]: pgmap v158: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-12T08:54:45.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 70 16 106468 106452 95.0346 56.125 0.021863 0.0105177 2024-04-12T08:54:46.052 INFO:teuthology.orchestra.run.smithi090.stdout:wait for half of osds to upgrade 2024-04-12T08:54:46.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 71 16 107401 107385 94.5173 58.3125 0.0117758 0.0105747 2024-04-12T08:54:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:46 smithi114 ceph-mon[89482]: pgmap v159: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s 2024-04-12T08:54:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:46 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:46 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:46.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:46 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:46.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:46 smithi090 ceph-mon[95814]: pgmap v159: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s 2024-04-12T08:54:46.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:46 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:46.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:46 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:46.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:46 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:46.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:46 smithi090 ceph-mon[99575]: pgmap v159: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s 2024-04-12T08:54:46.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:46 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:54:46.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:46 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:46.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:46 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:46.804 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-12T08:54:47.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 72 16 108310 108294 93.9936 56.8125 0.00854138 0.0106343 2024-04-12T08:54:47.287 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:54:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:54:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:54:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:54:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:54:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-12T08:54:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-12T08:54:47.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-12T08:54:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-12T08:54:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-12T08:54:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-12T08:54:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-12T08:54:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-12T08:54:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-12T08:54:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:54:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:47.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T08:54:47.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:47.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:54:47.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:47.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:54:47.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:54:47.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:47.425 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T08:54:47.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:47.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:54:47.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:47.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:54:47.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:54:47.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:54:47.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:47.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:47.426 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T08:54:48.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 73 16 109161 109145 93.4346 53.1875 0.015753 0.010697 2024-04-12T08:54:48.340 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:48 smithi114 ceph-mon[89482]: pgmap v160: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-12T08:54:48.341 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:48 smithi114 ceph-mon[89482]: Upgrade: Setting container_image for all mon 2024-04-12T08:54:48.341 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:48 smithi114 ceph-mon[89482]: Upgrade: Setting container_image for all crash 2024-04-12T08:54:48.341 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:48 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T08:54:48.341 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:48 smithi114 ceph-mon[89482]: Upgrade: osd.0 is safe to restart 2024-04-12T08:54:48.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:48 smithi090 ceph-mon[95814]: pgmap v160: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-12T08:54:48.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:48 smithi090 ceph-mon[95814]: Upgrade: Setting container_image for all mon 2024-04-12T08:54:48.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:48 smithi090 ceph-mon[95814]: Upgrade: Setting container_image for all crash 2024-04-12T08:54:48.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:48 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T08:54:48.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:48 smithi090 ceph-mon[95814]: Upgrade: osd.0 is safe to restart 2024-04-12T08:54:48.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:48 smithi090 ceph-mon[99575]: pgmap v160: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-12T08:54:48.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:48 smithi090 ceph-mon[99575]: Upgrade: Setting container_image for all mon 2024-04-12T08:54:48.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:48 smithi090 ceph-mon[99575]: Upgrade: Setting container_image for all crash 2024-04-12T08:54:48.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:48 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T08:54:48.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:48 smithi090 ceph-mon[99575]: Upgrade: osd.0 is safe to restart 2024-04-12T08:54:49.150 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 74 16 110077 110061 92.9455 57.25 0.0100592 0.0107543 2024-04-12T08:54:50.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 75 16 110931 110915 92.4179 53.375 0.0130418 0.0108152 2024-04-12T08:54:50.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:50 smithi114 ceph-mon[89482]: pgmap v161: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2024-04-12T08:54:50.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:50 smithi090 ceph-mon[99575]: pgmap v161: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2024-04-12T08:54:50.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:50 smithi090 ceph-mon[95814]: pgmap v161: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2024-04-12T08:54:51.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 76 16 111837 111821 91.9469 56.625 0.0189341 0.0108709 2024-04-12T08:54:51.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:51 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3039551710' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:51.372 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:51 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3039551710' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:51.372 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:51 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3039551710' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:52.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 77 16 112699 112683 91.4524 53.875 0.0332072 0.0109294 2024-04-12T08:54:52.669 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[95814]: pgmap v162: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s 2024-04-12T08:54:52.669 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[95814]: Upgrade: Updating osd.0 2024-04-12T08:54:52.669 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:52.669 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T08:54:52.670 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:52.670 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[95814]: Deploying daemon osd.0 on smithi090 2024-04-12T08:54:52.670 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[99575]: pgmap v162: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s 2024-04-12T08:54:52.670 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[99575]: Upgrade: Updating osd.0 2024-04-12T08:54:52.670 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:52.670 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T08:54:52.670 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:52.670 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:52 smithi090 ceph-mon[99575]: Deploying daemon osd.0 on smithi090 2024-04-12T08:54:52.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:52 smithi114 ceph-mon[89482]: pgmap v162: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s 2024-04-12T08:54:52.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:52 smithi114 ceph-mon[89482]: Upgrade: Updating osd.0 2024-04-12T08:54:52.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:52 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:54:52.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:52 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T08:54:52.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:52 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:54:52.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:52 smithi114 ceph-mon[89482]: Deploying daemon osd.0 on smithi090 2024-04-12T08:54:53.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 78 16 113543 113527 90.9562 52.75 0.0286162 0.0109889 2024-04-12T08:54:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:54:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:54:53] "GET /metrics HTTP/1.1" 200 36343 "" "Prometheus/2.43.0" 2024-04-12T08:54:53.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:53 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4279881573' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:53.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:53 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4279881573' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:53 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4279881573' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:54.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 79 16 114455 114439 90.5263 57 0.0292435 0.0110411 2024-04-12T08:54:54.821 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:54 smithi090 ceph-mon[95814]: pgmap v163: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-12T08:54:54.821 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:54 smithi090 ceph-mon[99575]: pgmap v163: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-12T08:54:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:54 smithi114 ceph-mon[89482]: pgmap v163: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-12T08:54:55.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:54:55.151535+0000 min lat: 0.000947928 max lat: 0.224002 avg lat: 0.0111012 2024-04-12T08:54:55.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:54:55.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 80 16 115279 115263 90.0384 51.5 0.0118739 0.0111012 2024-04-12T08:54:55.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:55 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/229148670' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:55.799 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:55 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/229148670' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:55.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:55 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/229148670' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:56.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 81 16 116121 116105 89.5765 52.625 0.0086122 0.0111588 2024-04-12T08:54:56.423 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:54:56 smithi090 systemd[1]: Stopping Ceph osd.0 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:54:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:56 smithi114 ceph-mon[89482]: pgmap v164: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 746 op/s 2024-04-12T08:54:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:56 smithi090 ceph-mon[95814]: pgmap v164: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 746 op/s 2024-04-12T08:54:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:56 smithi090 ceph-mon[99575]: pgmap v164: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 746 op/s 2024-04-12T08:54:57.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 82 16 116998 116982 89.1525 54.8125 0.0128429 0.0112113 2024-04-12T08:54:58.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 83 16 117825 117809 88.701 51.6875 0.00997819 0.011269 2024-04-12T08:54:58.173 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:54:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[43921]: 2024-04-12T08:54:57.730+0000 7fd5ee3ed700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T08:54:58.173 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:54:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[43921]: 2024-04-12T08:54:57.730+0000 7fd5ee3ed700 -1 osd.0 88 *** Got signal Terminated *** 2024-04-12T08:54:58.173 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:54:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[43921]: 2024-04-12T08:54:57.733+0000 7fd5ee3ed700 -1 osd.0 88 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T08:54:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:58 smithi114 ceph-mon[89482]: pgmap v165: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-04-12T08:54:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:58 smithi114 ceph-mon[89482]: osd.0 marked itself down and dead 2024-04-12T08:54:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:58 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3200867014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:58.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:58 smithi090 ceph-mon[99575]: pgmap v165: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-04-12T08:54:58.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:58 smithi090 ceph-mon[99575]: osd.0 marked itself down and dead 2024-04-12T08:54:58.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:58 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3200867014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:58 smithi090 ceph-mon[95814]: pgmap v165: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-04-12T08:54:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:58 smithi090 ceph-mon[95814]: osd.0 marked itself down and dead 2024-04-12T08:54:58.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:58 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3200867014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:54:59.151 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 84 16 118187 118171 87.9144 22.625 0.00301314 0.0112872 2024-04-12T08:54:59.391 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:54:59 smithi090 podman[107609]: 2024-04-12 08:54:59.127680102 +0000 UTC m=+2.710896819 container died 9395ed639a37b55f3eca4d0d2d9dfe6be7386f6bebf3dac10c4a1c48bdd6ec2b (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060) 2024-04-12T08:54:59.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:59 smithi090 ceph-mon[95814]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T08:54:59.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:54:59 smithi090 ceph-mon[95814]: osdmap e89: 8 total, 7 up, 8 in 2024-04-12T08:54:59.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:59 smithi090 ceph-mon[99575]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T08:54:59.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:54:59 smithi090 ceph-mon[99575]: osdmap e89: 8 total, 7 up, 8 in 2024-04-12T08:54:59.674 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:54:59 smithi090 podman[107609]: 2024-04-12 08:54:59.489869732 +0000 UTC m=+3.073086444 container cleanup 9395ed639a37b55f3eca4d0d2d9dfe6be7386f6bebf3dac10c4a1c48bdd6ec2b (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T08:54:59.674 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:54:59 smithi090 bash[107609]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0 2024-04-12T08:54:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:59 smithi114 ceph-mon[89482]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T08:54:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:54:59 smithi114 ceph-mon[89482]: osdmap e89: 8 total, 7 up, 8 in 2024-04-12T08:55:00.152 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 85 16 118850 118834 87.3676 41.4375 0.00929122 0.0114406 2024-04-12T08:55:00.173 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:54:59 smithi090 podman[107649]: 2024-04-12 08:54:59.810319647 +0000 UTC m=+0.687093632 container remove 9395ed639a37b55f3eca4d0d2d9dfe6be7386f6bebf3dac10c4a1c48bdd6ec2b (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0, RELEASE=quincy-d3b5060, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-04-12T08:55:00.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: pgmap v167: 89 pgs: 15 stale+active+clean, 74 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2024-04-12T08:55:00.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: osdmap e90: 8 total, 7 up, 8 in 2024-04-12T08:55:00.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]: dispatch 2024-04-12T08:55:00.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 1, 7, 0]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 1, 7, 0]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]: dispatch 2024-04-12T08:55:00.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2024-04-12T08:55:00.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "7.0", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2024-04-12T08:55:00.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "7.0", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:00.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:00.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: pgmap v167: 89 pgs: 15 stale+active+clean, 74 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2024-04-12T08:55:00.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: osdmap e90: 8 total, 7 up, 8 in 2024-04-12T08:55:00.676 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 1, 7, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 1, 7, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2024-04-12T08:55:00.677 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "7.0", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.678 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2024-04-12T08:55:00.678 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "7.0", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.678 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:00.678 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:00.678 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:00 smithi090 podman[107747]: 2024-04-12 08:55:00.261662042 +0000 UTC m=+0.035190583 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:55:00.678 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:00 smithi090 podman[107747]: 2024-04-12 08:55:00.41471665 +0000 UTC m=+0.188245173 container create 406f0a773b769a98457891dc1db002daf736304a31f1fa92cc87206ab8f609e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:55:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: pgmap v167: 89 pgs: 15 stale+active+clean, 74 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2024-04-12T08:55:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: osdmap e90: 8 total, 7 up, 8 in 2024-04-12T08:55:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]: dispatch 2024-04-12T08:55:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 1, 7, 0]}]: dispatch 2024-04-12T08:55:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]: dispatch 2024-04-12T08:55:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]: dispatch 2024-04-12T08:55:00.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 1, 7, 0]}]: dispatch 2024-04-12T08:55:00.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]: dispatch 2024-04-12T08:55:00.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]: dispatch 2024-04-12T08:55:00.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2024-04-12T08:55:00.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "7.0", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2024-04-12T08:55:00.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "7.0", "id": [3, 0]}]: dispatch 2024-04-12T08:55:00.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:00.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:01.100 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:00 smithi090 podman[107747]: 2024-04-12 08:55:00.709141798 +0000 UTC m=+0.482670322 container init 406f0a773b769a98457891dc1db002daf736304a31f1fa92cc87206ab8f609e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=) 2024-04-12T08:55:01.100 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:00 smithi090 podman[107747]: 2024-04-12 08:55:00.714087254 +0000 UTC m=+0.487615779 container start 406f0a773b769a98457891dc1db002daf736304a31f1fa92cc87206ab8f609e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:55:01.100 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:00 smithi090 podman[107747]: 2024-04-12 08:55:00.82481292 +0000 UTC m=+0.598341448 container attach 406f0a773b769a98457891dc1db002daf736304a31f1fa92cc87206ab8f609e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0) 2024-04-12T08:55:01.152 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 86 16 119638 119622 86.9243 49.25 0.00353833 0.0114572 2024-04-12T08:55:01.422 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:01 smithi090 podman[107747]: 2024-04-12 08:55:01.100763176 +0000 UTC m=+0.874291701 container died 406f0a773b769a98457891dc1db002daf736304a31f1fa92cc87206ab8f609e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:55:01.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-04-12T08:55:01.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2024-04-12T08:55:01.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]': finished 2024-04-12T08:55:01.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 1, 7, 0]}]': finished 2024-04-12T08:55:01.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [3, 0]}]': finished 2024-04-12T08:55:01.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]': finished 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]': finished 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "7.0", "id": [3, 0]}]': finished 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: osdmap e91: 8 total, 7 up, 8 in 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3562769400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]': finished 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 1, 7, 0]}]': finished 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [3, 0]}]': finished 2024-04-12T08:55:01.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]': finished 2024-04-12T08:55:01.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-04-12T08:55:01.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]': finished 2024-04-12T08:55:01.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "7.0", "id": [3, 0]}]': finished 2024-04-12T08:55:01.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: osdmap e91: 8 total, 7 up, 8 in 2024-04-12T08:55:01.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:01 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3562769400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-04-12T08:55:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2024-04-12T08:55:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]': finished 2024-04-12T08:55:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 1, 7, 0]}]': finished 2024-04-12T08:55:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [3, 0]}]': finished 2024-04-12T08:55:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]': finished 2024-04-12T08:55:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-04-12T08:55:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]': finished 2024-04-12T08:55:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "7.0", "id": [3, 0]}]': finished 2024-04-12T08:55:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: osdmap e91: 8 total, 7 up, 8 in 2024-04-12T08:55:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:01 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3562769400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:02.152 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 87 16 119638 119622 85.9252 0 - 0.0114572 2024-04-12T08:55:02.423 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:02 smithi090 podman[107775]: 2024-04-12 08:55:02.082153766 +0000 UTC m=+0.971722970 container remove 406f0a773b769a98457891dc1db002daf736304a31f1fa92cc87206ab8f609e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:55:02.423 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:02 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-12T08:55:02.423 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:02 smithi090 systemd[1]: Stopped Ceph osd.0 for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:55:02.423 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:02 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.0.service: Consumed 15min 31.864s CPU time. 2024-04-12T08:55:02.424 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:02 smithi090 systemd[1]: Starting Ceph osd.0 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:55:02.782 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:02 smithi090 ceph-mon[99575]: pgmap v170: 89 pgs: 15 stale+active+clean, 74 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s 2024-04-12T08:55:02.783 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:02 smithi090 ceph-mon[99575]: osdmap e92: 8 total, 7 up, 8 in 2024-04-12T08:55:02.783 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:02 smithi090 ceph-mon[95814]: pgmap v170: 89 pgs: 15 stale+active+clean, 74 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s 2024-04-12T08:55:02.783 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:02 smithi090 ceph-mon[95814]: osdmap e92: 8 total, 7 up, 8 in 2024-04-12T08:55:02.783 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:02 smithi090 podman[107880]: 2024-04-12 08:55:02.41664868 +0000 UTC m=+0.017709691 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:55:02.783 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:02 smithi090 podman[107880]: 2024-04-12 08:55:02.54209165 +0000 UTC m=+0.143152661 container create ddabb251ed2a9d35ea2c4160469e731a6ba13c9881bc12df2ff2bed4945c6d02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-12T08:55:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:02 smithi114 ceph-mon[89482]: pgmap v170: 89 pgs: 15 stale+active+clean, 74 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s 2024-04-12T08:55:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:02 smithi114 ceph-mon[89482]: osdmap e92: 8 total, 7 up, 8 in 2024-04-12T08:55:03.152 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 88 16 120138 120122 85.3039 15.625 0.00790437 0.0117171 2024-04-12T08:55:03.173 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:02 smithi090 podman[107880]: 2024-04-12 08:55:02.782656028 +0000 UTC m=+0.383717051 container init ddabb251ed2a9d35ea2c4160469e731a6ba13c9881bc12df2ff2bed4945c6d02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T08:55:03.173 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:02 smithi090 podman[107880]: 2024-04-12 08:55:02.793322812 +0000 UTC m=+0.394383833 container start ddabb251ed2a9d35ea2c4160469e731a6ba13c9881bc12df2ff2bed4945c6d02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-12T08:55:03.173 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:02 smithi090 podman[107880]: 2024-04-12 08:55:02.880897947 +0000 UTC m=+0.481958968 container attach ddabb251ed2a9d35ea2c4160469e731a6ba13c9881bc12df2ff2bed4945c6d02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-b277feb) 2024-04-12T08:55:03.545 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:55:03] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2024-04-12T08:55:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:03 smithi114 ceph-mon[89482]: osdmap e93: 8 total, 7 up, 8 in 2024-04-12T08:55:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:03 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3913039033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-mon[99575]: osdmap e93: 8 total, 7 up, 8 in 2024-04-12T08:55:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3913039033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:03.923 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate[107898]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T08:55:03.923 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate[107898]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T08:55:03.923 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate[107898]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate[107898]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate[107898]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate[107898]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate[107898]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 bash[107880]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 bash[107880]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 bash[107880]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 bash[107880]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 bash[107880]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 bash[107880]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 bash[107880]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 conmon[107898]: conmon ddabb251ed2a9d35ea2c : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-ddabb251ed2a9d35ea2c4160469e731a6ba13c9881bc12df2ff2bed4945c6d02.scope/container/memory.events 2024-04-12T08:55:03.924 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:03 smithi090 podman[107880]: 2024-04-12 08:55:03.604741807 +0000 UTC m=+1.205802824 container died ddabb251ed2a9d35ea2c4160469e731a6ba13c9881bc12df2ff2bed4945c6d02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-12T08:55:03.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-mon[95814]: osdmap e93: 8 total, 7 up, 8 in 2024-04-12T08:55:03.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:03 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3913039033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:04.152 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 89 16 121281 121265 85.148 71.4375 0.00665906 0.0117399 2024-04-12T08:55:04.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:04 smithi090 ceph-mon[99575]: pgmap v173: 89 pgs: 4 remapped+peering, 4 peering, 14 active+undersized+degraded, 18 active+undersized, 49 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s; 59750/358938 objects degraded (16.646%) 2024-04-12T08:55:04.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:04 smithi090 ceph-mon[99575]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-12T08:55:04.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:04 smithi090 ceph-mon[99575]: Health check failed: Degraded data redundancy: 59750/358938 objects degraded (16.646%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:04.674 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:04 smithi090 podman[108044]: 2024-04-12 08:55:04.406416051 +0000 UTC m=+0.788482349 container remove ddabb251ed2a9d35ea2c4160469e731a6ba13c9881bc12df2ff2bed4945c6d02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-12T08:55:04.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:04 smithi090 ceph-mon[95814]: pgmap v173: 89 pgs: 4 remapped+peering, 4 peering, 14 active+undersized+degraded, 18 active+undersized, 49 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s; 59750/358938 objects degraded (16.646%) 2024-04-12T08:55:04.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:04 smithi090 ceph-mon[95814]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-12T08:55:04.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:04 smithi090 ceph-mon[95814]: Health check failed: Degraded data redundancy: 59750/358938 objects degraded (16.646%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:04.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:04 smithi114 ceph-mon[89482]: pgmap v173: 89 pgs: 4 remapped+peering, 4 peering, 14 active+undersized+degraded, 18 active+undersized, 49 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s; 59750/358938 objects degraded (16.646%) 2024-04-12T08:55:04.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:04 smithi114 ceph-mon[89482]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-12T08:55:04.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:04 smithi114 ceph-mon[89482]: Health check failed: Degraded data redundancy: 59750/358938 objects degraded (16.646%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:05.041 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:04 smithi090 podman[108096]: 2024-04-12 08:55:04.654651015 +0000 UTC m=+0.043778201 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:55:05.041 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:04 smithi090 podman[108096]: 2024-04-12 08:55:04.783156644 +0000 UTC m=+0.172283861 container create c245fffdfa5671d4b8e6ff5579dab5e815a9c1af8d4459a626c88d5ad8a2b94a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T08:55:05.041 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:05 smithi090 podman[108096]: 2024-04-12 08:55:05.040599191 +0000 UTC m=+0.429726369 container init c245fffdfa5671d4b8e6ff5579dab5e815a9c1af8d4459a626c88d5ad8a2b94a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-12T08:55:05.152 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Total time run: 90.006 2024-04-12T08:55:05.152 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Total writes made: 122420 2024-04-12T08:55:05.152 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Write size: 65536 2024-04-12T08:55:05.152 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Object size: 65536 2024-04-12T08:55:05.152 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Bandwidth (MB/sec): 85.0082 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev Bandwidth: 40.1476 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max bandwidth (MB/sec): 216.125 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min bandwidth (MB/sec): 0 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Average IOPS: 1360 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev IOPS: 642.362 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max IOPS: 3458 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min IOPS: 0 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Average Latency(s): 0.0117596 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev Latency(s): 0.0256485 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max latency(s): 2.01081 2024-04-12T08:55:05.153 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min latency(s): 0.000947928 2024-04-12T08:55:05.157 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Cleaning up (deleting benchmark objects) 2024-04-12T08:55:05.423 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:05 smithi090 podman[108096]: 2024-04-12 08:55:05.045713205 +0000 UTC m=+0.434840375 container start c245fffdfa5671d4b8e6ff5579dab5e815a9c1af8d4459a626c88d5ad8a2b94a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T08:55:05.423 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:05 smithi090 bash[108096]: c245fffdfa5671d4b8e6ff5579dab5e815a9c1af8d4459a626c88d5ad8a2b94a 2024-04-12T08:55:05.423 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:05 smithi090 systemd[1]: Started Ceph osd.0 for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:55:05.923 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:05 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[108111]: 2024-04-12T08:55:05.611+0000 7f3e7cc2a7c0 -1 Falling back to public interface 2024-04-12T08:55:06.605 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:06 smithi090 ceph-mon[99575]: pgmap v174: 89 pgs: 4 remapped+peering, 4 peering, 14 active+undersized+degraded, 18 active+undersized, 49 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 36 MiB/s wr, 573 op/s; 59750/358938 objects degraded (16.646%) 2024-04-12T08:55:06.605 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:06 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:06.605 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:06 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:06.605 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:06 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:55:06.605 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:06 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1261543003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:06.605 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:06 smithi090 ceph-mon[95814]: pgmap v174: 89 pgs: 4 remapped+peering, 4 peering, 14 active+undersized+degraded, 18 active+undersized, 49 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 36 MiB/s wr, 573 op/s; 59750/358938 objects degraded (16.646%) 2024-04-12T08:55:06.605 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:06 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:06.606 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:06 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:06.606 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:06 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:55:06.606 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:06 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1261543003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:06.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:06 smithi114 ceph-mon[89482]: pgmap v174: 89 pgs: 4 remapped+peering, 4 peering, 14 active+undersized+degraded, 18 active+undersized, 49 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 36 MiB/s wr, 573 op/s; 59750/358938 objects degraded (16.646%) 2024-04-12T08:55:06.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:06 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:06.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:06 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:06.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:06 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:55:06.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:06 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1261543003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:08.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:08 smithi114 ceph-mon[89482]: pgmap v175: 89 pgs: 7 active+clean+remapped, 2 remapped+peering, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 165 B/s rd, 48 MiB/s wr, 902 op/s; 60245/361554 objects degraded (16.663%); 7552/361554 objects misplaced (2.089%); 0 B/s, 0 objects/s recovering 2024-04-12T08:55:08.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:08 smithi090 ceph-mon[99575]: pgmap v175: 89 pgs: 7 active+clean+remapped, 2 remapped+peering, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 165 B/s rd, 48 MiB/s wr, 902 op/s; 60245/361554 objects degraded (16.663%); 7552/361554 objects misplaced (2.089%); 0 B/s, 0 objects/s recovering 2024-04-12T08:55:08.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:08 smithi090 ceph-mon[95814]: pgmap v175: 89 pgs: 7 active+clean+remapped, 2 remapped+peering, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 165 B/s rd, 48 MiB/s wr, 902 op/s; 60245/361554 objects degraded (16.663%); 7552/361554 objects misplaced (2.089%); 0 B/s, 0 objects/s recovering 2024-04-12T08:55:09.577 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:09 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2074910803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:09.577 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:09 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2074910803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:09.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:09 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2074910803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:10.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:10 smithi114 ceph-mon[89482]: pgmap v176: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 45 MiB/s wr, 925 op/s; 60357/362490 objects degraded (16.651%); 7554/362490 objects misplaced (2.084%); 11 B/s, 0 objects/s recovering 2024-04-12T08:55:10.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:10 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 60357/362490 objects degraded (16.651%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:10.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:10 smithi114 ceph-mon[89482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-12T08:55:10.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:10 smithi090 ceph-mon[95814]: pgmap v176: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 45 MiB/s wr, 925 op/s; 60357/362490 objects degraded (16.651%); 7554/362490 objects misplaced (2.084%); 11 B/s, 0 objects/s recovering 2024-04-12T08:55:10.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:10 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 60357/362490 objects degraded (16.651%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:10.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:10 smithi090 ceph-mon[95814]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-12T08:55:10.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:10 smithi090 ceph-mon[99575]: pgmap v176: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 45 MiB/s wr, 925 op/s; 60357/362490 objects degraded (16.651%); 7554/362490 objects misplaced (2.084%); 11 B/s, 0 objects/s recovering 2024-04-12T08:55:10.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:10 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 60357/362490 objects degraded (16.651%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:10.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:10 smithi090 ceph-mon[99575]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-12T08:55:12.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:11 smithi114 ceph-mon[89482]: pgmap v177: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 111 B/s rd, 29 MiB/s wr, 639 op/s; 60357/362490 objects degraded (16.651%); 7554/362490 objects misplaced (2.084%); 9 B/s, 0 objects/s recovering 2024-04-12T08:55:12.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:11 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:12.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:11 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:12.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:11 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2371082114' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:11 smithi090 ceph-mon[95814]: pgmap v177: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 111 B/s rd, 29 MiB/s wr, 639 op/s; 60357/362490 objects degraded (16.651%); 7554/362490 objects misplaced (2.084%); 9 B/s, 0 objects/s recovering 2024-04-12T08:55:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:11 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:11 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:11 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2371082114' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:11 smithi090 ceph-mon[99575]: pgmap v177: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 111 B/s rd, 29 MiB/s wr, 639 op/s; 60357/362490 objects degraded (16.651%); 7554/362490 objects misplaced (2.084%); 9 B/s, 0 objects/s recovering 2024-04-12T08:55:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:11 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:11 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:11 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2371082114' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:55:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:55:13] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2024-04-12T08:55:14.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:13 smithi114 ceph-mon[89482]: pgmap v178: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 100 B/s rd, 17 MiB/s wr, 929 op/s; 57830/347397 objects degraded (16.647%); 7249/347397 objects misplaced (2.087%); 8 B/s, 0 objects/s recovering 2024-04-12T08:55:14.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:13 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:14.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:13 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:14.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:13 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/465633611' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:14.170 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:13 smithi090 ceph-mon[95814]: pgmap v178: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 100 B/s rd, 17 MiB/s wr, 929 op/s; 57830/347397 objects degraded (16.647%); 7249/347397 objects misplaced (2.087%); 8 B/s, 0 objects/s recovering 2024-04-12T08:55:14.170 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:14.170 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:14.170 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:13 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/465633611' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:14.170 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:13 smithi090 ceph-mon[99575]: pgmap v178: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 100 B/s rd, 17 MiB/s wr, 929 op/s; 57830/347397 objects degraded (16.647%); 7249/347397 objects misplaced (2.087%); 8 B/s, 0 objects/s recovering 2024-04-12T08:55:14.171 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:13 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:14.171 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:13 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:14.171 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:13 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/465633611' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:14.774 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:14 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[108111]: 2024-04-12T08:55:14.626+0000 7f3e7cc2a7c0 -1 osd.0 88 log_to_monitors true 2024-04-12T08:55:15.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:14 smithi114 ceph-mon[89482]: from='osd.0 [v2:172.21.15.90:6802/3578880330,v1:172.21.15.90:6803/3578880330]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:55:15.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:14 smithi114 ceph-mon[89482]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:55:15.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:14 smithi090 ceph-mon[99575]: from='osd.0 [v2:172.21.15.90:6802/3578880330,v1:172.21.15.90:6803/3578880330]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:55:15.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:14 smithi090 ceph-mon[99575]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:55:15.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:14 smithi090 ceph-mon[95814]: from='osd.0 [v2:172.21.15.90:6802/3578880330,v1:172.21.15.90:6803/3578880330]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:55:15.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:14 smithi090 ceph-mon[95814]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T08:55:15.596 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[108111]: 2024-04-12T08:55:15.327+0000 7f3e6a07a700 -1 osd.0 88 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T08:55:15.850 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[99575]: pgmap v179: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 786 op/s; 57830/347397 objects degraded (16.647%); 7249/347397 objects misplaced (2.087%); 7 B/s, 0 objects/s recovering 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[99575]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[99575]: osdmap e94: 8 total, 7 up, 8 in 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[99575]: from='osd.0 [v2:172.21.15.90:6802/3578880330,v1:172.21.15.90:6803/3578880330]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[99575]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[99575]: from='osd.0 ' entity='osd.0' 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 57830/347397 objects degraded (16.647%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[95814]: pgmap v179: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 786 op/s; 57830/347397 objects degraded (16.647%); 7249/347397 objects misplaced (2.087%); 7 B/s, 0 objects/s recovering 2024-04-12T08:55:15.851 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[95814]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T08:55:15.852 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[95814]: osdmap e94: 8 total, 7 up, 8 in 2024-04-12T08:55:15.852 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[95814]: from='osd.0 [v2:172.21.15.90:6802/3578880330,v1:172.21.15.90:6803/3578880330]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:55:15.852 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[95814]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:55:15.852 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[95814]: from='osd.0 ' entity='osd.0' 2024-04-12T08:55:15.852 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 57830/347397 objects degraded (16.647%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:15.852 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:15.852 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:15.852 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:15 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:16.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:15 smithi114 ceph-mon[89482]: pgmap v179: 89 pgs: 9 active+clean+remapped, 14 active+undersized+degraded, 18 active+undersized, 48 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 786 op/s; 57830/347397 objects degraded (16.647%); 7249/347397 objects misplaced (2.087%); 7 B/s, 0 objects/s recovering 2024-04-12T08:55:16.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:15 smithi114 ceph-mon[89482]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T08:55:16.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:15 smithi114 ceph-mon[89482]: osdmap e94: 8 total, 7 up, 8 in 2024-04-12T08:55:16.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:15 smithi114 ceph-mon[89482]: from='osd.0 [v2:172.21.15.90:6802/3578880330,v1:172.21.15.90:6803/3578880330]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:55:16.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:15 smithi114 ceph-mon[89482]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:55:16.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:15 smithi114 ceph-mon[89482]: from='osd.0 ' entity='osd.0' 2024-04-12T08:55:16.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:15 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 57830/347397 objects degraded (16.647%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:16.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:15 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:16.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:15 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:16.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:15 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:16 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4164568222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:16 smithi114 ceph-mon[89482]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T08:55:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:16 smithi114 ceph-mon[89482]: osd.0 [v2:172.21.15.90:6802/3578880330,v1:172.21.15.90:6803/3578880330] boot 2024-04-12T08:55:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:16 smithi114 ceph-mon[89482]: osdmap e95: 8 total, 8 up, 8 in 2024-04-12T08:55:17.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:16 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:55:17.129 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:16 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4164568222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:17.129 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:16 smithi090 ceph-mon[99575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T08:55:17.129 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:16 smithi090 ceph-mon[99575]: osd.0 [v2:172.21.15.90:6802/3578880330,v1:172.21.15.90:6803/3578880330] boot 2024-04-12T08:55:17.129 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:16 smithi090 ceph-mon[99575]: osdmap e95: 8 total, 8 up, 8 in 2024-04-12T08:55:17.129 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:16 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:55:17.129 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:16 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4164568222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:17.129 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:16 smithi090 ceph-mon[95814]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T08:55:17.130 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:16 smithi090 ceph-mon[95814]: osd.0 [v2:172.21.15.90:6802/3578880330,v1:172.21.15.90:6803/3578880330] boot 2024-04-12T08:55:17.130 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:16 smithi090 ceph-mon[95814]: osdmap e95: 8 total, 8 up, 8 in 2024-04-12T08:55:17.130 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T08:55:18.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:18 smithi090 ceph-mon[99575]: pgmap v182: 89 pgs: 7 remapped+peering, 2 active+clean+remapped, 12 peering, 8 active+undersized+degraded, 12 active+undersized, 48 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 36168/341532 objects degraded (10.590%); 2/341532 objects misplaced (0.001%) 2024-04-12T08:55:18.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:18 smithi090 ceph-mon[99575]: osdmap e96: 8 total, 8 up, 8 in 2024-04-12T08:55:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:18 smithi090 ceph-mon[95814]: pgmap v182: 89 pgs: 7 remapped+peering, 2 active+clean+remapped, 12 peering, 8 active+undersized+degraded, 12 active+undersized, 48 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 36168/341532 objects degraded (10.590%); 2/341532 objects misplaced (0.001%) 2024-04-12T08:55:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:18 smithi090 ceph-mon[95814]: osdmap e96: 8 total, 8 up, 8 in 2024-04-12T08:55:18.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:18 smithi114 ceph-mon[89482]: pgmap v182: 89 pgs: 7 remapped+peering, 2 active+clean+remapped, 12 peering, 8 active+undersized+degraded, 12 active+undersized, 48 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 36168/341532 objects degraded (10.590%); 2/341532 objects misplaced (0.001%) 2024-04-12T08:55:18.755 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:18 smithi114 ceph-mon[89482]: osdmap e96: 8 total, 8 up, 8 in 2024-04-12T08:55:19.351 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:55:19.003+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-04-12T08:55:19.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: osdmap e97: 8 total, 8 up, 8 in 2024-04-12T08:55:19.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/184356705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:19.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:19.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:19.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:55:19.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:55:19.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:19.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:55:19.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:55:19.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:55:19.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:55:19.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:19 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:19.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: osdmap e97: 8 total, 8 up, 8 in 2024-04-12T08:55:19.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/184356705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:19.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:19.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:19.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:55:19.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:55:19.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:19.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:55:19.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:55:19.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:55:19.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:55:19.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:19.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: osdmap e97: 8 total, 8 up, 8 in 2024-04-12T08:55:19.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/184356705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:19.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:19.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:19.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:55:19.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:55:19.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:19.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:55:19.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:55:19.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:55:19.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:55:19.676 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:19 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:20 smithi114 ceph-mon[89482]: pgmap v185: 89 pgs: 9 remapped+peering, 18 peering, 5 active+undersized+degraded, 9 active+undersized, 48 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 28979/337845 objects degraded (8.578%) 2024-04-12T08:55:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:20 smithi114 ceph-mon[89482]: pgmap v186: 89 pgs: 9 remapped+peering, 18 peering, 5 active+undersized+degraded, 9 active+undersized, 48 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 28979/337845 objects degraded (8.578%) 2024-04-12T08:55:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:20 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:20.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:20 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-04-12T08:55:20.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:20 smithi090 ceph-mon[99575]: pgmap v185: 89 pgs: 9 remapped+peering, 18 peering, 5 active+undersized+degraded, 9 active+undersized, 48 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 28979/337845 objects degraded (8.578%) 2024-04-12T08:55:20.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:20 smithi090 ceph-mon[99575]: pgmap v186: 89 pgs: 9 remapped+peering, 18 peering, 5 active+undersized+degraded, 9 active+undersized, 48 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 28979/337845 objects degraded (8.578%) 2024-04-12T08:55:20.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:20 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:20.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:20 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-04-12T08:55:20.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:20 smithi090 ceph-mon[95814]: pgmap v185: 89 pgs: 9 remapped+peering, 18 peering, 5 active+undersized+degraded, 9 active+undersized, 48 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 28979/337845 objects degraded (8.578%) 2024-04-12T08:55:20.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:20 smithi090 ceph-mon[95814]: pgmap v186: 89 pgs: 9 remapped+peering, 18 peering, 5 active+undersized+degraded, 9 active+undersized, 48 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 28979/337845 objects degraded (8.578%) 2024-04-12T08:55:20.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:20 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:20.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:20 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-04-12T08:55:21.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:21 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 28979/337845 objects degraded (8.578%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:21.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:21 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1675786507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:21.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:21 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 28979/337845 objects degraded (8.578%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:21.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:21 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1675786507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:21.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:21 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 28979/337845 objects degraded (8.578%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:21.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:21 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1675786507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:22.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:22 smithi114 ceph-mon[89482]: pgmap v187: 89 pgs: 6 remapped+peering, 18 peering, 4 active+undersized+degraded, 61 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 28977/337845 objects degraded (8.577%); 1.0 KiB/s, 0 objects/s recovering 2024-04-12T08:55:22.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:22 smithi090 ceph-mon[95814]: pgmap v187: 89 pgs: 6 remapped+peering, 18 peering, 4 active+undersized+degraded, 61 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 28977/337845 objects degraded (8.577%); 1.0 KiB/s, 0 objects/s recovering 2024-04-12T08:55:22.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:22 smithi090 ceph-mon[99575]: pgmap v187: 89 pgs: 6 remapped+peering, 18 peering, 4 active+undersized+degraded, 61 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 28977/337845 objects degraded (8.577%); 1.0 KiB/s, 0 objects/s recovering 2024-04-12T08:55:23.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:55:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:55:23] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.43.0" 2024-04-12T08:55:24.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:24 smithi090 ceph-mon[95814]: pgmap v188: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 14319/327216 objects degraded (4.376%); 1.7 MiB/s, 28 objects/s recovering 2024-04-12T08:55:24.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:24 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3682091314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:24 smithi090 ceph-mon[99575]: pgmap v188: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 14319/327216 objects degraded (4.376%); 1.7 MiB/s, 28 objects/s recovering 2024-04-12T08:55:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:24 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3682091314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:24 smithi114 ceph-mon[89482]: pgmap v188: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 14319/327216 objects degraded (4.376%); 1.7 MiB/s, 28 objects/s recovering 2024-04-12T08:55:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:24 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3682091314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:26 smithi090 ceph-mon[95814]: pgmap v189: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 14319/327216 objects degraded (4.376%); 1.4 MiB/s, 23 objects/s recovering 2024-04-12T08:55:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:26 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 14319/327216 objects degraded (4.376%), 9 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:26 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/792412887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:26 smithi090 ceph-mon[99575]: pgmap v189: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 14319/327216 objects degraded (4.376%); 1.4 MiB/s, 23 objects/s recovering 2024-04-12T08:55:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:26 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 14319/327216 objects degraded (4.376%), 9 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:26 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/792412887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:26 smithi114 ceph-mon[89482]: pgmap v189: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 14319/327216 objects degraded (4.376%); 1.4 MiB/s, 23 objects/s recovering 2024-04-12T08:55:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:26 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 14319/327216 objects degraded (4.376%), 9 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:26 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/792412887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:28.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:28 smithi090 ceph-mon[95814]: pgmap v190: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 14220/320625 objects degraded (4.435%); 1.9 MiB/s, 55 objects/s recovering 2024-04-12T08:55:28.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:28 smithi090 ceph-mon[99575]: pgmap v190: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 14220/320625 objects degraded (4.435%); 1.9 MiB/s, 55 objects/s recovering 2024-04-12T08:55:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:28 smithi114 ceph-mon[89482]: pgmap v190: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 14220/320625 objects degraded (4.435%); 1.9 MiB/s, 55 objects/s recovering 2024-04-12T08:55:29.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:29 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3004710337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:29.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:29 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3004710337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:29.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:29 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3004710337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:30.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:30 smithi090 ceph-mon[95814]: pgmap v191: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 14220/315531 objects degraded (4.507%); 1.6 MiB/s, 46 objects/s recovering 2024-04-12T08:55:30.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:30 smithi090 ceph-mon[99575]: pgmap v191: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 14220/315531 objects degraded (4.507%); 1.6 MiB/s, 46 objects/s recovering 2024-04-12T08:55:30.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:30 smithi114 ceph-mon[89482]: pgmap v191: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 14220/315531 objects degraded (4.507%); 1.6 MiB/s, 46 objects/s recovering 2024-04-12T08:55:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:31 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 14220/315531 objects degraded (4.507%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:31 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:31 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 14220/315531 objects degraded (4.507%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:31 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:31 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 14220/315531 objects degraded (4.507%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:31 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:55:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:32 smithi114 ceph-mon[89482]: pgmap v192: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 14220/315531 objects degraded (4.507%); 1.4 MiB/s, 38 objects/s recovering 2024-04-12T08:55:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:32 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1667751320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:32.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:32 smithi090 ceph-mon[95814]: pgmap v192: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 14220/315531 objects degraded (4.507%); 1.4 MiB/s, 38 objects/s recovering 2024-04-12T08:55:32.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:32 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1667751320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:32.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:32 smithi090 ceph-mon[99575]: pgmap v192: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 14220/315531 objects degraded (4.507%); 1.4 MiB/s, 38 objects/s recovering 2024-04-12T08:55:32.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:32 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1667751320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:55:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:55:33] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2024-04-12T08:55:34.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:55:34 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:55:34.021+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:55:34.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:34 smithi114 ceph-mon[89482]: pgmap v193: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 14220/304071 objects degraded (4.677%); 1.4 MiB/s, 61 objects/s recovering 2024-04-12T08:55:34.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:34 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:34.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:34 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3891223669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:34.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:34 smithi090 ceph-mon[95814]: pgmap v193: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 14220/304071 objects degraded (4.677%); 1.4 MiB/s, 61 objects/s recovering 2024-04-12T08:55:34.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:34 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:34.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:34 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3891223669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:34.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:34 smithi090 ceph-mon[99575]: pgmap v193: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 14220/304071 objects degraded (4.677%); 1.4 MiB/s, 61 objects/s recovering 2024-04-12T08:55:34.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:34 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:34.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:34 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3891223669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:35 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:35 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:55:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:35 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 14220/304071 objects degraded (4.677%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:35 smithi114 ceph-mon[89482]: osdmap e98: 8 total, 8 up, 8 in 2024-04-12T08:55:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:35 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:35 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:55:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:35 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 14220/304071 objects degraded (4.677%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:35 smithi090 ceph-mon[95814]: osdmap e98: 8 total, 8 up, 8 in 2024-04-12T08:55:35.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:35 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:35.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:35 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:55:35.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:35 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 14220/304071 objects degraded (4.677%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:35.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:35 smithi090 ceph-mon[99575]: osdmap e98: 8 total, 8 up, 8 in 2024-04-12T08:55:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:36 smithi114 ceph-mon[89482]: pgmap v194: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 14220/304071 objects degraded (4.677%); 528 KiB/s, 47 objects/s recovering 2024-04-12T08:55:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:36 smithi114 ceph-mon[89482]: osdmap e99: 8 total, 8 up, 8 in 2024-04-12T08:55:36.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:36 smithi090 ceph-mon[95814]: pgmap v194: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 14220/304071 objects degraded (4.677%); 528 KiB/s, 47 objects/s recovering 2024-04-12T08:55:36.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:36 smithi090 ceph-mon[95814]: osdmap e99: 8 total, 8 up, 8 in 2024-04-12T08:55:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:36 smithi090 ceph-mon[99575]: pgmap v194: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 14220/304071 objects degraded (4.677%); 528 KiB/s, 47 objects/s recovering 2024-04-12T08:55:36.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:36 smithi090 ceph-mon[99575]: osdmap e99: 8 total, 8 up, 8 in 2024-04-12T08:55:37.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:37 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3542185370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:37.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:37 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3542185370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:37.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:37 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3542185370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:38.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:38 smithi114 ceph-mon[89482]: pgmap v197: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 14220/298521 objects degraded (4.763%); 0 B/s, 61 objects/s recovering 2024-04-12T08:55:38.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:38 smithi090 ceph-mon[99575]: pgmap v197: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 14220/298521 objects degraded (4.763%); 0 B/s, 61 objects/s recovering 2024-04-12T08:55:38.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:38 smithi090 ceph-mon[95814]: pgmap v197: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 14220/298521 objects degraded (4.763%); 0 B/s, 61 objects/s recovering 2024-04-12T08:55:39.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:39 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/280377675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:39.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:39 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/280377675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:39.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:39 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/280377675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:40.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:40 smithi114 ceph-mon[89482]: pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 13294/294138 objects degraded (4.520%); 2.0 MiB/s, 94 objects/s recovering 2024-04-12T08:55:40.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:40 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 13294/294138 objects degraded (4.520%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:40 smithi090 ceph-mon[99575]: pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 13294/294138 objects degraded (4.520%); 2.0 MiB/s, 94 objects/s recovering 2024-04-12T08:55:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:40 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 13294/294138 objects degraded (4.520%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:40 smithi090 ceph-mon[95814]: pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 13294/294138 objects degraded (4.520%); 2.0 MiB/s, 94 objects/s recovering 2024-04-12T08:55:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:40 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 13294/294138 objects degraded (4.520%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:42.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:42 smithi114 ceph-mon[89482]: pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 13294/294138 objects degraded (4.520%); 2.0 MiB/s, 59 objects/s recovering 2024-04-12T08:55:42.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:42 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4109626947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:42 smithi090 ceph-mon[99575]: pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 13294/294138 objects degraded (4.520%); 2.0 MiB/s, 59 objects/s recovering 2024-04-12T08:55:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:42 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4109626947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:42.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:42 smithi090 ceph-mon[95814]: pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 13294/294138 objects degraded (4.520%); 2.0 MiB/s, 59 objects/s recovering 2024-04-12T08:55:42.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:42 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4109626947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:43.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:55:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:55:43] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2024-04-12T08:55:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:44 smithi114 ceph-mon[89482]: pgmap v200: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 13283/282795 objects degraded (4.697%); 2.1 MiB/s, 88 objects/s recovering 2024-04-12T08:55:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:44 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3990260017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:44 smithi090 ceph-mon[99575]: pgmap v200: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 13283/282795 objects degraded (4.697%); 2.1 MiB/s, 88 objects/s recovering 2024-04-12T08:55:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:44 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3990260017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:44 smithi090 ceph-mon[95814]: pgmap v200: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 13283/282795 objects degraded (4.697%); 2.1 MiB/s, 88 objects/s recovering 2024-04-12T08:55:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:44 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3990260017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:45 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 13283/282795 objects degraded (4.697%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:45 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 13283/282795 objects degraded (4.697%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:45 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 13283/282795 objects degraded (4.697%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:55:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:46 smithi114 ceph-mon[89482]: pgmap v201: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 13283/282795 objects degraded (4.697%); 1.8 MiB/s, 75 objects/s recovering 2024-04-12T08:55:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:46 smithi090 ceph-mon[99575]: pgmap v201: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 13283/282795 objects degraded (4.697%); 1.8 MiB/s, 75 objects/s recovering 2024-04-12T08:55:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:46 smithi090 ceph-mon[95814]: pgmap v201: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 13283/282795 objects degraded (4.697%); 1.8 MiB/s, 75 objects/s recovering 2024-04-12T08:55:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:47 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1087037455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:47 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1087037455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:47 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1087037455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:48.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:48 smithi114 ceph-mon[89482]: pgmap v202: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 13283/276297 objects degraded (4.808%); 1.6 MiB/s, 68 objects/s recovering 2024-04-12T08:55:48.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:48 smithi090 ceph-mon[99575]: pgmap v202: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 13283/276297 objects degraded (4.808%); 1.6 MiB/s, 68 objects/s recovering 2024-04-12T08:55:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:48 smithi090 ceph-mon[95814]: pgmap v202: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 13283/276297 objects degraded (4.808%); 1.6 MiB/s, 68 objects/s recovering 2024-04-12T08:55:49.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:55:49 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:55:49.038+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:55:49.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:49 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:49.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:49 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/364634381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:49 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:49 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/364634381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:49 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:49 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/364634381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:50 smithi114 ceph-mon[89482]: pgmap v203: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 13283/271422 objects degraded (4.894%); 1.4 MiB/s, 62 objects/s recovering 2024-04-12T08:55:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:50 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:50 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:55:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:50 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 13283/271422 objects degraded (4.894%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:50 smithi090 ceph-mon[95814]: pgmap v203: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 13283/271422 objects degraded (4.894%); 1.4 MiB/s, 62 objects/s recovering 2024-04-12T08:55:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:50 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:50 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:55:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:50 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 13283/271422 objects degraded (4.894%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:50 smithi090 ceph-mon[99575]: pgmap v203: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 13283/271422 objects degraded (4.894%); 1.4 MiB/s, 62 objects/s recovering 2024-04-12T08:55:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:50 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:55:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:50 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:55:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:50 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 13283/271422 objects degraded (4.894%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:52.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:52 smithi114 ceph-mon[89482]: pgmap v204: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 13283/271422 objects degraded (4.894%); 59 KiB/s, 40 objects/s recovering 2024-04-12T08:55:52.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:52 smithi114 ceph-mon[89482]: osdmap e100: 8 total, 8 up, 8 in 2024-04-12T08:55:52.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:52 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2930980781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:52.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:52 smithi090 ceph-mon[95814]: pgmap v204: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 13283/271422 objects degraded (4.894%); 59 KiB/s, 40 objects/s recovering 2024-04-12T08:55:52.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:52 smithi090 ceph-mon[95814]: osdmap e100: 8 total, 8 up, 8 in 2024-04-12T08:55:52.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:52 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2930980781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:52.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:52 smithi090 ceph-mon[99575]: pgmap v204: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 13283/271422 objects degraded (4.894%); 59 KiB/s, 40 objects/s recovering 2024-04-12T08:55:52.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:52 smithi090 ceph-mon[99575]: osdmap e100: 8 total, 8 up, 8 in 2024-04-12T08:55:52.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:52 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2930980781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:53.625 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:55:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:55:53] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2024-04-12T08:55:53.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:53 smithi090 ceph-mon[99575]: osdmap e101: 8 total, 8 up, 8 in 2024-04-12T08:55:53.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:53 smithi090 ceph-mon[95814]: osdmap e101: 8 total, 8 up, 8 in 2024-04-12T08:55:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:53 smithi114 ceph-mon[89482]: osdmap e101: 8 total, 8 up, 8 in 2024-04-12T08:55:54.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:54 smithi090 ceph-mon[95814]: pgmap v207: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 12512/261183 objects degraded (4.791%); 416 KiB/s, 38 objects/s recovering 2024-04-12T08:55:54.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:54 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/786310411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:54.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:54 smithi090 ceph-mon[99575]: pgmap v207: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 12512/261183 objects degraded (4.791%); 416 KiB/s, 38 objects/s recovering 2024-04-12T08:55:54.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:54 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/786310411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:55.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:54 smithi114 ceph-mon[89482]: pgmap v207: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 12512/261183 objects degraded (4.791%); 416 KiB/s, 38 objects/s recovering 2024-04-12T08:55:55.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:54 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/786310411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:55 smithi090 ceph-mon[95814]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 12512/261183 objects degraded (4.791%); 416 KiB/s, 38 objects/s recovering 2024-04-12T08:55:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:55 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 12512/261183 objects degraded (4.791%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:55 smithi090 ceph-mon[99575]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 12512/261183 objects degraded (4.791%); 416 KiB/s, 38 objects/s recovering 2024-04-12T08:55:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:55 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 12512/261183 objects degraded (4.791%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:55 smithi114 ceph-mon[89482]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 12512/261183 objects degraded (4.791%); 416 KiB/s, 38 objects/s recovering 2024-04-12T08:55:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:55 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 12512/261183 objects degraded (4.791%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:55:58.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:57 smithi090 ceph-mon[99575]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 12297/254862 objects degraded (4.825%); 2.1 MiB/s, 87 objects/s recovering 2024-04-12T08:55:58.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:57 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2098813293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:58.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:57 smithi090 ceph-mon[95814]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 12297/254862 objects degraded (4.825%); 2.1 MiB/s, 87 objects/s recovering 2024-04-12T08:55:58.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:57 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2098813293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:55:58.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:57 smithi114 ceph-mon[89482]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 12297/254862 objects degraded (4.825%); 2.1 MiB/s, 87 objects/s recovering 2024-04-12T08:55:58.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:57 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2098813293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:00.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:59 smithi090 ceph-mon[95814]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 12297/251160 objects degraded (4.896%); 2.1 MiB/s, 87 objects/s recovering 2024-04-12T08:56:00.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:55:59 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3037834156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:00.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:59 smithi090 ceph-mon[99575]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 12297/251160 objects degraded (4.896%); 2.1 MiB/s, 87 objects/s recovering 2024-04-12T08:56:00.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:55:59 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3037834156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:00.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:59 smithi114 ceph-mon[89482]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 12297/251160 objects degraded (4.896%); 2.1 MiB/s, 87 objects/s recovering 2024-04-12T08:56:00.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:55:59 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3037834156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:00 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 12297/251160 objects degraded (4.896%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:00 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 12297/251160 objects degraded (4.896%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:01.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:00 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 12297/251160 objects degraded (4.896%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:01.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:02.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:01 smithi090 ceph-mon[95814]: pgmap v211: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 12297/250428 objects degraded (4.910%); 1.8 MiB/s, 75 objects/s recovering 2024-04-12T08:56:02.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:01 smithi090 ceph-mon[99575]: pgmap v211: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 12297/250428 objects degraded (4.910%); 1.8 MiB/s, 75 objects/s recovering 2024-04-12T08:56:02.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:01 smithi114 ceph-mon[89482]: pgmap v211: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 12297/250428 objects degraded (4.910%); 1.8 MiB/s, 75 objects/s recovering 2024-04-12T08:56:03.280 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:02 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/292603743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:03.280 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:02 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/292603743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:03.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:02 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/292603743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:03.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:56:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:56:03] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.43.0" 2024-04-12T08:56:04.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:03 smithi114 ceph-mon[89482]: pgmap v212: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 12297/239718 objects degraded (5.130%); 1.3 MiB/s, 90 objects/s recovering 2024-04-12T08:56:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:03 smithi090 ceph-mon[95814]: pgmap v212: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 12297/239718 objects degraded (5.130%); 1.3 MiB/s, 90 objects/s recovering 2024-04-12T08:56:04.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:56:04 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:56:04.055+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:56:04.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:03 smithi090 ceph-mon[99575]: pgmap v212: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 12297/239718 objects degraded (5.130%); 1.3 MiB/s, 90 objects/s recovering 2024-04-12T08:56:05.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:04 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:05.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:04 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:05.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:04 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:56:05.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:04 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1390500031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:05.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:04 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:05.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:04 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:05.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:04 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:56:05.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:04 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1390500031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:05.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:04 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:05.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:04 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:05.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:04 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:56:05.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:04 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1390500031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:05 smithi114 ceph-mon[89482]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12297/239718 objects degraded (5.130%); 1.1 MiB/s, 76 objects/s recovering 2024-04-12T08:56:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:05 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 12297/239718 objects degraded (5.130%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:06.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:05 smithi114 ceph-mon[89482]: osdmap e102: 8 total, 8 up, 8 in 2024-04-12T08:56:06.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:05 smithi090 ceph-mon[95814]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12297/239718 objects degraded (5.130%); 1.1 MiB/s, 76 objects/s recovering 2024-04-12T08:56:06.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:05 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 12297/239718 objects degraded (5.130%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:06.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:05 smithi090 ceph-mon[95814]: osdmap e102: 8 total, 8 up, 8 in 2024-04-12T08:56:06.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:05 smithi090 ceph-mon[99575]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12297/239718 objects degraded (5.130%); 1.1 MiB/s, 76 objects/s recovering 2024-04-12T08:56:06.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:05 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 12297/239718 objects degraded (5.130%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:06.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:05 smithi090 ceph-mon[99575]: osdmap e102: 8 total, 8 up, 8 in 2024-04-12T08:56:07.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:07 smithi090 ceph-mon[95814]: osdmap e103: 8 total, 8 up, 8 in 2024-04-12T08:56:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:07 smithi090 ceph-mon[99575]: osdmap e103: 8 total, 8 up, 8 in 2024-04-12T08:56:07.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:07 smithi114 ceph-mon[89482]: osdmap e103: 8 total, 8 up, 8 in 2024-04-12T08:56:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:08 smithi114 ceph-mon[89482]: pgmap v216: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 11417/234444 objects degraded (4.870%); 1.7 MiB/s, 61 objects/s recovering 2024-04-12T08:56:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:08 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1480056674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:08.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:08 smithi090 ceph-mon[99575]: pgmap v216: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 11417/234444 objects degraded (4.870%); 1.7 MiB/s, 61 objects/s recovering 2024-04-12T08:56:08.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:08 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1480056674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:08.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:08 smithi090 ceph-mon[95814]: pgmap v216: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 11417/234444 objects degraded (4.870%); 1.7 MiB/s, 61 objects/s recovering 2024-04-12T08:56:08.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:08 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1480056674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:10 smithi114 ceph-mon[89482]: pgmap v217: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 11417/230775 objects degraded (4.947%); 1.7 MiB/s, 61 objects/s recovering 2024-04-12T08:56:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:10 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2075547744' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:10 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 11417/230775 objects degraded (4.947%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:10 smithi090 ceph-mon[99575]: pgmap v217: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 11417/230775 objects degraded (4.947%); 1.7 MiB/s, 61 objects/s recovering 2024-04-12T08:56:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:10 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2075547744' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:10 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 11417/230775 objects degraded (4.947%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:10 smithi090 ceph-mon[95814]: pgmap v217: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 11417/230775 objects degraded (4.947%); 1.7 MiB/s, 61 objects/s recovering 2024-04-12T08:56:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:10 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2075547744' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:10 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 11417/230775 objects degraded (4.947%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:12 smithi114 ceph-mon[89482]: pgmap v218: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 11417/229128 objects degraded (4.983%); 1.7 MiB/s, 50 objects/s recovering 2024-04-12T08:56:12.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:12 smithi090 ceph-mon[99575]: pgmap v218: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 11417/229128 objects degraded (4.983%); 1.7 MiB/s, 50 objects/s recovering 2024-04-12T08:56:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:12 smithi090 ceph-mon[95814]: pgmap v218: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 11417/229128 objects degraded (4.983%); 1.7 MiB/s, 50 objects/s recovering 2024-04-12T08:56:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:56:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:56:13] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.43.0" 2024-04-12T08:56:13.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:13 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1561467480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:13.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:13 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1561467480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:13 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1561467480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:14.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:14 smithi114 ceph-mon[89482]: pgmap v219: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 11352/219108 objects degraded (5.181%); 2.2 MiB/s, 82 objects/s recovering 2024-04-12T08:56:14.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:14 smithi090 ceph-mon[99575]: pgmap v219: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 11352/219108 objects degraded (5.181%); 2.2 MiB/s, 82 objects/s recovering 2024-04-12T08:56:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:14 smithi090 ceph-mon[95814]: pgmap v219: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 11352/219108 objects degraded (5.181%); 2.2 MiB/s, 82 objects/s recovering 2024-04-12T08:56:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:15 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2369246002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:15 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 11352/219108 objects degraded (5.181%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:15 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2369246002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:15 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 11352/219108 objects degraded (5.181%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:15 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2369246002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:15 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 11352/219108 objects degraded (5.181%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T08:56:16.768 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:16 smithi114 ceph-mon[89482]: pgmap v220: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 11352/219108 objects degraded (5.181%); 1.8 MiB/s, 70 objects/s recovering 2024-04-12T08:56:16.768 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:16 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:16 smithi090 ceph-mon[99575]: pgmap v220: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 11352/219108 objects degraded (5.181%); 1.8 MiB/s, 70 objects/s recovering 2024-04-12T08:56:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:16 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:16 smithi090 ceph-mon[95814]: pgmap v220: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 11352/219108 objects degraded (5.181%); 1.8 MiB/s, 70 objects/s recovering 2024-04-12T08:56:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:18.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:18 smithi114 ceph-mon[89482]: pgmap v221: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 11352/211992 objects degraded (5.355%); 1.7 MiB/s, 87 objects/s recovering 2024-04-12T08:56:18.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:18 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2350643786' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:18.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:18 smithi090 ceph-mon[99575]: pgmap v221: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 11352/211992 objects degraded (5.355%); 1.7 MiB/s, 87 objects/s recovering 2024-04-12T08:56:18.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:18 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2350643786' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:18.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:18 smithi090 ceph-mon[95814]: pgmap v221: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 11352/211992 objects degraded (5.355%); 1.7 MiB/s, 87 objects/s recovering 2024-04-12T08:56:18.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:18 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2350643786' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:19.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:19 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:56:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:19 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:56:19.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:19 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:56:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:20 smithi114 ceph-mon[89482]: pgmap v222: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 11352/207816 objects degraded (5.463%); 341 KiB/s, 58 objects/s recovering 2024-04-12T08:56:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:20 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 11352/207816 objects degraded (5.463%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:20 smithi114 ceph-mon[89482]: osdmap e104: 8 total, 8 up, 8 in 2024-04-12T08:56:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:20 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3494864990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:20 smithi090 ceph-mon[99575]: pgmap v222: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 11352/207816 objects degraded (5.463%); 341 KiB/s, 58 objects/s recovering 2024-04-12T08:56:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:20 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 11352/207816 objects degraded (5.463%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:20 smithi090 ceph-mon[99575]: osdmap e104: 8 total, 8 up, 8 in 2024-04-12T08:56:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:20 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3494864990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:20 smithi090 ceph-mon[95814]: pgmap v222: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 11352/207816 objects degraded (5.463%); 341 KiB/s, 58 objects/s recovering 2024-04-12T08:56:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:20 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 11352/207816 objects degraded (5.463%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:20.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:20 smithi090 ceph-mon[95814]: osdmap e104: 8 total, 8 up, 8 in 2024-04-12T08:56:20.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:20 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3494864990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:22.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:22 smithi090 ceph-mon[95814]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 11352/206463 objects degraded (5.498%); 409 KiB/s, 50 objects/s recovering 2024-04-12T08:56:22.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:22 smithi090 ceph-mon[95814]: osdmap e105: 8 total, 8 up, 8 in 2024-04-12T08:56:22.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:22 smithi090 ceph-mon[99575]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 11352/206463 objects degraded (5.498%); 409 KiB/s, 50 objects/s recovering 2024-04-12T08:56:22.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:22 smithi090 ceph-mon[99575]: osdmap e105: 8 total, 8 up, 8 in 2024-04-12T08:56:22.811 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:22 smithi114 ceph-mon[89482]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 11352/206463 objects degraded (5.498%); 409 KiB/s, 50 objects/s recovering 2024-04-12T08:56:22.812 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:22 smithi114 ceph-mon[89482]: osdmap e105: 8 total, 8 up, 8 in 2024-04-12T08:56:23.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:23 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2652032163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:23.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:23 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2652032163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:23.675 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:56:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:56:23] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.43.0" 2024-04-12T08:56:23.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:23 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2652032163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:24 smithi090 ceph-mon[99575]: pgmap v226: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 10521/197994 objects degraded (5.314%); 1.2 MiB/s, 78 objects/s recovering 2024-04-12T08:56:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:24 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:24 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:24.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:24 smithi090 ceph-mon[95814]: pgmap v226: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 10521/197994 objects degraded (5.314%); 1.2 MiB/s, 78 objects/s recovering 2024-04-12T08:56:24.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:24 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:24.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:24 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:24 smithi114 ceph-mon[89482]: pgmap v226: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 10521/197994 objects degraded (5.314%); 1.2 MiB/s, 78 objects/s recovering 2024-04-12T08:56:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:24 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:24 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:25 smithi090 ceph-mon[95814]: pgmap v227: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 10521/197994 objects degraded (5.314%); 1.2 MiB/s, 47 objects/s recovering 2024-04-12T08:56:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:25 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:25 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:25 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 10521/197994 objects degraded (5.314%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:26.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2146405887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:25 smithi090 ceph-mon[99575]: pgmap v227: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 10521/197994 objects degraded (5.314%); 1.2 MiB/s, 47 objects/s recovering 2024-04-12T08:56:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:25 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:25 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:25 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 10521/197994 objects degraded (5.314%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:26.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2146405887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:25 smithi114 ceph-mon[89482]: pgmap v227: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 10521/197994 objects degraded (5.314%); 1.2 MiB/s, 47 objects/s recovering 2024-04-12T08:56:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:25 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:25 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:25 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 10521/197994 objects degraded (5.314%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:26.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2146405887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:26.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:56:26.172+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:56:27.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:56:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: pgmap v228: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 10521/196863 objects degraded (5.344%); 1.3 MiB/s, 53 objects/s recovering 2024-04-12T08:56:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:56:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:56:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:56:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:56:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:27.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: pgmap v228: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 10521/196863 objects degraded (5.344%); 1.3 MiB/s, 53 objects/s recovering 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:27.175 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:26 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:27.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:56:27.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:56:27.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: pgmap v228: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 10521/196863 objects degraded (5.344%); 1.3 MiB/s, 53 objects/s recovering 2024-04-12T08:56:27.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:56:27.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:56:27.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:56:27.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:56:27.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:56:27.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:27.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:27.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:26 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:29.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:29 smithi114 ceph-mon[89482]: pgmap v229: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 10403/188418 objects degraded (5.521%); 2.2 MiB/s, 74 objects/s recovering 2024-04-12T08:56:29.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:29 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/742198753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:29.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:29 smithi090 ceph-mon[99575]: pgmap v229: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 10403/188418 objects degraded (5.521%); 2.2 MiB/s, 74 objects/s recovering 2024-04-12T08:56:29.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:29 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/742198753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:29.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:29 smithi090 ceph-mon[95814]: pgmap v229: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 10403/188418 objects degraded (5.521%); 2.2 MiB/s, 74 objects/s recovering 2024-04-12T08:56:29.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:29 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/742198753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:31 smithi114 ceph-mon[89482]: pgmap v230: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 10403/188418 objects degraded (5.521%); 1.9 MiB/s, 62 objects/s recovering 2024-04-12T08:56:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:31 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 10403/188418 objects degraded (5.521%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:31 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3518719240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:31 smithi090 ceph-mon[99575]: pgmap v230: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 10403/188418 objects degraded (5.521%); 1.9 MiB/s, 62 objects/s recovering 2024-04-12T08:56:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:31 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 10403/188418 objects degraded (5.521%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:31 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3518719240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:31 smithi090 ceph-mon[95814]: pgmap v230: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 10403/188418 objects degraded (5.521%); 1.9 MiB/s, 62 objects/s recovering 2024-04-12T08:56:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:31 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 10403/188418 objects degraded (5.521%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:31 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3518719240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:33.280 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:33 smithi090 ceph-mon[99575]: pgmap v231: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 10403/181515 objects degraded (5.731%); 1.6 MiB/s, 53 objects/s recovering 2024-04-12T08:56:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:33 smithi090 ceph-mon[95814]: pgmap v231: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 10403/181515 objects degraded (5.731%); 1.6 MiB/s, 53 objects/s recovering 2024-04-12T08:56:33.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:33 smithi114 ceph-mon[89482]: pgmap v231: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 10403/181515 objects degraded (5.731%); 1.6 MiB/s, 53 objects/s recovering 2024-04-12T08:56:33.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:56:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:56:33] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.43.0" 2024-04-12T08:56:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:34 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3949240304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:34.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:34 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3949240304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:34.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:34 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3949240304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:35.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:35 smithi090 ceph-mon[99575]: pgmap v232: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 10403/177420 objects degraded (5.863%); 681 KiB/s, 36 objects/s recovering 2024-04-12T08:56:35.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:35 smithi090 ceph-mon[95814]: pgmap v232: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 10403/177420 objects degraded (5.863%); 681 KiB/s, 36 objects/s recovering 2024-04-12T08:56:35.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:35 smithi114 ceph-mon[89482]: pgmap v232: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 10403/177420 objects degraded (5.863%); 681 KiB/s, 36 objects/s recovering 2024-04-12T08:56:36.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:36 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 10403/177420 objects degraded (5.863%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:36.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:36 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 10403/177420 objects degraded (5.863%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:36.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:36 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 10403/177420 objects degraded (5.863%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:37.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:37 smithi114 ceph-mon[89482]: pgmap v233: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 10403/176016 objects degraded (5.910%); 681 KiB/s, 36 objects/s recovering 2024-04-12T08:56:37.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:37 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4174360636' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:37.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:37 smithi090 ceph-mon[95814]: pgmap v233: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 10403/176016 objects degraded (5.910%); 681 KiB/s, 36 objects/s recovering 2024-04-12T08:56:37.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:37 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4174360636' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:37.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:37 smithi090 ceph-mon[99575]: pgmap v233: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 10403/176016 objects degraded (5.910%); 681 KiB/s, 36 objects/s recovering 2024-04-12T08:56:37.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:37 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4174360636' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:39.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:39 smithi114 ceph-mon[89482]: pgmap v234: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 10403/166233 objects degraded (6.258%); 629 KiB/s, 51 objects/s recovering 2024-04-12T08:56:39.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:39 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1853296368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:39.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:39 smithi090 ceph-mon[95814]: pgmap v234: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 10403/166233 objects degraded (6.258%); 629 KiB/s, 51 objects/s recovering 2024-04-12T08:56:39.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:39 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1853296368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:39.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:39 smithi090 ceph-mon[99575]: pgmap v234: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 10403/166233 objects degraded (6.258%); 629 KiB/s, 51 objects/s recovering 2024-04-12T08:56:39.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:39 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1853296368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:41.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:41 smithi114 ceph-mon[89482]: pgmap v235: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10403/166233 objects degraded (6.258%); 0 B/s, 35 objects/s recovering 2024-04-12T08:56:41.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:41 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 10403/166233 objects degraded (6.258%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:41 smithi090 ceph-mon[95814]: pgmap v235: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10403/166233 objects degraded (6.258%); 0 B/s, 35 objects/s recovering 2024-04-12T08:56:41.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:41 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 10403/166233 objects degraded (6.258%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:41.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:56:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:56:41.188+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:41.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:41 smithi090 ceph-mon[99575]: pgmap v235: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10403/166233 objects degraded (6.258%); 0 B/s, 35 objects/s recovering 2024-04-12T08:56:41.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:41 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 10403/166233 objects degraded (6.258%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:42 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:42 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:42 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:42 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2878672259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:42.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:42 smithi114 ceph-mon[89482]: osdmap e106: 8 total, 8 up, 8 in 2024-04-12T08:56:42.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:42 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:42.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:42 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:42.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:42 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:42.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:42 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2878672259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:42.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:42 smithi090 ceph-mon[99575]: osdmap e106: 8 total, 8 up, 8 in 2024-04-12T08:56:42.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:42 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:42.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:42 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:42.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:42 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:42.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:42 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2878672259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:42.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:42 smithi090 ceph-mon[95814]: osdmap e106: 8 total, 8 up, 8 in 2024-04-12T08:56:43.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:56:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:56:43] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.43.0" 2024-04-12T08:56:43.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:43 smithi090 ceph-mon[99575]: pgmap v237: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 6478/159213 objects degraded (4.069%); 3265/159213 objects misplaced (2.051%); 64 KiB/s, 43 objects/s recovering 2024-04-12T08:56:43.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:43 smithi090 ceph-mon[99575]: osdmap e107: 8 total, 8 up, 8 in 2024-04-12T08:56:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:43 smithi090 ceph-mon[95814]: pgmap v237: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 6478/159213 objects degraded (4.069%); 3265/159213 objects misplaced (2.051%); 64 KiB/s, 43 objects/s recovering 2024-04-12T08:56:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:43 smithi090 ceph-mon[95814]: osdmap e107: 8 total, 8 up, 8 in 2024-04-12T08:56:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:43 smithi114 ceph-mon[89482]: pgmap v237: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 6478/159213 objects degraded (4.069%); 3265/159213 objects misplaced (2.051%); 64 KiB/s, 43 objects/s recovering 2024-04-12T08:56:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:43 smithi114 ceph-mon[89482]: osdmap e107: 8 total, 8 up, 8 in 2024-04-12T08:56:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:44 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2049282913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:44 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2049282913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:44 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2049282913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:45 smithi114 ceph-mon[89482]: pgmap v239: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 6478/155643 objects degraded (4.162%); 3265/155643 objects misplaced (2.098%); 80 KiB/s, 49 objects/s recovering 2024-04-12T08:56:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:45 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 6478/155643 objects degraded (4.162%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:45 smithi090 ceph-mon[95814]: pgmap v239: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 6478/155643 objects degraded (4.162%); 3265/155643 objects misplaced (2.098%); 80 KiB/s, 49 objects/s recovering 2024-04-12T08:56:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:45 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 6478/155643 objects degraded (4.162%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:45 smithi090 ceph-mon[99575]: pgmap v239: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 6478/155643 objects degraded (4.162%); 3265/155643 objects misplaced (2.098%); 80 KiB/s, 49 objects/s recovering 2024-04-12T08:56:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:45 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 6478/155643 objects degraded (4.162%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:46.767 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:46 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:46 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:46 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:56:47.781 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:47 smithi114 ceph-mon[89482]: pgmap v240: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 6478/154425 objects degraded (4.195%); 3265/154425 objects misplaced (2.114%); 80 KiB/s, 23 objects/s recovering 2024-04-12T08:56:47.781 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:47 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3783149731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:47 smithi090 ceph-mon[95814]: pgmap v240: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 6478/154425 objects degraded (4.195%); 3265/154425 objects misplaced (2.114%); 80 KiB/s, 23 objects/s recovering 2024-04-12T08:56:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:47 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3783149731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:47 smithi090 ceph-mon[99575]: pgmap v240: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 6478/154425 objects degraded (4.195%); 3265/154425 objects misplaced (2.114%); 80 KiB/s, 23 objects/s recovering 2024-04-12T08:56:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:47 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3783149731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:49.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:49 smithi114 ceph-mon[89482]: pgmap v241: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 6557/145665 objects degraded (4.501%); 3061/145665 objects misplaced (2.101%); 1.1 MiB/s, 39 objects/s recovering 2024-04-12T08:56:49.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:49 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2651965454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:49 smithi090 ceph-mon[95814]: pgmap v241: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 6557/145665 objects degraded (4.501%); 3061/145665 objects misplaced (2.101%); 1.1 MiB/s, 39 objects/s recovering 2024-04-12T08:56:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:49 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2651965454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:49 smithi090 ceph-mon[99575]: pgmap v241: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 6557/145665 objects degraded (4.501%); 3061/145665 objects misplaced (2.101%); 1.1 MiB/s, 39 objects/s recovering 2024-04-12T08:56:49.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:49 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2651965454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:50 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 6557/145665 objects degraded (4.501%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:50 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 6557/145665 objects degraded (4.501%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:50 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 6557/145665 objects degraded (4.501%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:51.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:51 smithi114 ceph-mon[89482]: pgmap v242: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 6557/145665 objects degraded (4.501%); 3061/145665 objects misplaced (2.101%); 1014 KiB/s, 37 objects/s recovering 2024-04-12T08:56:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:51 smithi090 ceph-mon[99575]: pgmap v242: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 6557/145665 objects degraded (4.501%); 3061/145665 objects misplaced (2.101%); 1014 KiB/s, 37 objects/s recovering 2024-04-12T08:56:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:51 smithi090 ceph-mon[95814]: pgmap v242: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 6557/145665 objects degraded (4.501%); 3061/145665 objects misplaced (2.101%); 1014 KiB/s, 37 objects/s recovering 2024-04-12T08:56:52.824 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:52 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/949579177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:52.825 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:52 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/949579177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:52.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:52 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/949579177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:56:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:56:53] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.43.0" 2024-04-12T08:56:53.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:53 smithi090 ceph-mon[99575]: pgmap v243: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 6712/138576 objects degraded (4.844%); 2823/138576 objects misplaced (2.037%); 1.3 MiB/s, 39 objects/s recovering 2024-04-12T08:56:53.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:53 smithi090 ceph-mon[95814]: pgmap v243: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 6712/138576 objects degraded (4.844%); 2823/138576 objects misplaced (2.037%); 1.3 MiB/s, 39 objects/s recovering 2024-04-12T08:56:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:53 smithi114 ceph-mon[89482]: pgmap v243: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 6712/138576 objects degraded (4.844%); 2823/138576 objects misplaced (2.037%); 1.3 MiB/s, 39 objects/s recovering 2024-04-12T08:56:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:54 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1484007815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:54.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:54 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1484007815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:54.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:54 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1484007815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:55.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:55 smithi114 ceph-mon[89482]: pgmap v244: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 6712/134301 objects degraded (4.998%); 2823/134301 objects misplaced (2.102%); 1.1 MiB/s, 18 objects/s recovering 2024-04-12T08:56:55.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:55 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 6712/134301 objects degraded (4.998%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:55 smithi090 ceph-mon[99575]: pgmap v244: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 6712/134301 objects degraded (4.998%); 2823/134301 objects misplaced (2.102%); 1.1 MiB/s, 18 objects/s recovering 2024-04-12T08:56:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:55 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 6712/134301 objects degraded (4.998%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:55 smithi090 ceph-mon[95814]: pgmap v244: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 6712/134301 objects degraded (4.998%); 2823/134301 objects misplaced (2.102%); 1.1 MiB/s, 18 objects/s recovering 2024-04-12T08:56:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:55 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 6712/134301 objects degraded (4.998%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:56:56.527 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:56:56 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:56:56.207+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:56 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:56 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:56 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:57 smithi114 ceph-mon[89482]: pgmap v245: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 6712/132876 objects degraded (5.051%); 2823/132876 objects misplaced (2.125%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:56:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:57 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:57 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:57 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3951027546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:57 smithi090 ceph-mon[95814]: pgmap v245: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 6712/132876 objects degraded (5.051%); 2823/132876 objects misplaced (2.125%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:56:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:57 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:57 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:57 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3951027546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:57.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:57 smithi090 ceph-mon[99575]: pgmap v245: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 6712/132876 objects degraded (5.051%); 2823/132876 objects misplaced (2.125%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:56:57.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:57 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:56:57.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:57 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:56:57.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:57 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3951027546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:59 smithi114 ceph-mon[89482]: pgmap v246: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 6861/123435 objects degraded (5.558%); 2590/123435 objects misplaced (2.098%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T08:56:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:56:59 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3959730750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:59.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:59 smithi090 ceph-mon[95814]: pgmap v246: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 6861/123435 objects degraded (5.558%); 2590/123435 objects misplaced (2.098%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T08:56:59.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:56:59 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3959730750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:56:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:59 smithi090 ceph-mon[99575]: pgmap v246: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 6861/123435 objects degraded (5.558%); 2590/123435 objects misplaced (2.098%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T08:56:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:56:59 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3959730750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:00 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 6861/123435 objects degraded (5.558%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:00 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 6861/123435 objects degraded (5.558%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:00.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:00 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 6861/123435 objects degraded (5.558%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:00.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:01 smithi114 ceph-mon[89482]: pgmap v247: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 6861/123435 objects degraded (5.558%); 2590/123435 objects misplaced (2.098%); 890 KiB/s, 13 objects/s recovering 2024-04-12T08:57:01.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:01 smithi090 ceph-mon[99575]: pgmap v247: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 6861/123435 objects degraded (5.558%); 2590/123435 objects misplaced (2.098%); 890 KiB/s, 13 objects/s recovering 2024-04-12T08:57:01.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:01 smithi090 ceph-mon[95814]: pgmap v247: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 6861/123435 objects degraded (5.558%); 2590/123435 objects misplaced (2.098%); 890 KiB/s, 13 objects/s recovering 2024-04-12T08:57:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:02 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4282269158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:02 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4282269158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:02 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4282269158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:03.562 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:57:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:57:03] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2024-04-12T08:57:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:03 smithi114 ceph-mon[89482]: pgmap v248: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 6973/116448 objects degraded (5.988%); 2378/116448 objects misplaced (2.042%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:03 smithi090 ceph-mon[99575]: pgmap v248: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 6973/116448 objects degraded (5.988%); 2378/116448 objects misplaced (2.042%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:03.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:03 smithi090 ceph-mon[95814]: pgmap v248: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 6973/116448 objects degraded (5.988%); 2378/116448 objects misplaced (2.042%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:05.854 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:05 smithi090 ceph-mon[99575]: pgmap v249: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 6973/112014 objects degraded (6.225%); 2378/112014 objects misplaced (2.123%); 981 KiB/s, 15 objects/s recovering 2024-04-12T08:57:05.854 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:05 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3680703172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:05.854 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:05 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 6973/112014 objects degraded (6.225%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:05.855 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:05 smithi090 ceph-mon[95814]: pgmap v249: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 6973/112014 objects degraded (6.225%); 2378/112014 objects misplaced (2.123%); 981 KiB/s, 15 objects/s recovering 2024-04-12T08:57:05.855 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:05 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3680703172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:05.855 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:05 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 6973/112014 objects degraded (6.225%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:05 smithi114 ceph-mon[89482]: pgmap v249: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 6973/112014 objects degraded (6.225%); 2378/112014 objects misplaced (2.123%); 981 KiB/s, 15 objects/s recovering 2024-04-12T08:57:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:05 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3680703172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:05 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 6973/112014 objects degraded (6.225%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:07.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:07 smithi114 ceph-mon[89482]: pgmap v250: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 6973/110550 objects degraded (6.308%); 2378/110550 objects misplaced (2.151%); 981 KiB/s, 15 objects/s recovering 2024-04-12T08:57:07.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:07 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3629574194' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:07.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:07 smithi090 ceph-mon[99575]: pgmap v250: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 6973/110550 objects degraded (6.308%); 2378/110550 objects misplaced (2.151%); 981 KiB/s, 15 objects/s recovering 2024-04-12T08:57:07.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:07 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3629574194' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:07.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:07 smithi090 ceph-mon[95814]: pgmap v250: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 6973/110550 objects degraded (6.308%); 2378/110550 objects misplaced (2.151%); 981 KiB/s, 15 objects/s recovering 2024-04-12T08:57:07.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:07 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3629574194' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:09.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:09 smithi114 ceph-mon[89482]: pgmap v251: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 7125/100677 objects degraded (7.077%); 2138/100677 objects misplaced (2.124%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:09.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:09 smithi090 ceph-mon[99575]: pgmap v251: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 7125/100677 objects degraded (7.077%); 2138/100677 objects misplaced (2.124%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:09.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:09 smithi090 ceph-mon[95814]: pgmap v251: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 7125/100677 objects degraded (7.077%); 2138/100677 objects misplaced (2.124%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:10 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2456471233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:10 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7125/100677 objects degraded (7.077%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:10 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2456471233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:10 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7125/100677 objects degraded (7.077%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:10 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2456471233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:10 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7125/100677 objects degraded (7.077%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:11.589 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:57:11 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:57:11.223+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:11 smithi114 ceph-mon[89482]: pgmap v252: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 7125/100677 objects degraded (7.077%); 2138/100677 objects misplaced (2.124%); 1002 KiB/s, 15 objects/s recovering 2024-04-12T08:57:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:11 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:11.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:11 smithi090 ceph-mon[99575]: pgmap v252: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 7125/100677 objects degraded (7.077%); 2138/100677 objects misplaced (2.124%); 1002 KiB/s, 15 objects/s recovering 2024-04-12T08:57:11.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:11.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:11 smithi090 ceph-mon[95814]: pgmap v252: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 7125/100677 objects degraded (7.077%); 2138/100677 objects misplaced (2.124%); 1002 KiB/s, 15 objects/s recovering 2024-04-12T08:57:11.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:12 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:12 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:12 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2329718995' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:12.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:12 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:12.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:12 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:12.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:12 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2329718995' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:12 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:12 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:12 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2329718995' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:13.597 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:57:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:57:13] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2024-04-12T08:57:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:13 smithi114 ceph-mon[89482]: pgmap v253: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 7298/93576 objects degraded (7.799%); 1881/93576 objects misplaced (2.010%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:13.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:13 smithi090 ceph-mon[99575]: pgmap v253: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 7298/93576 objects degraded (7.799%); 1881/93576 objects misplaced (2.010%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:13.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:13 smithi090 ceph-mon[95814]: pgmap v253: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 7298/93576 objects degraded (7.799%); 1881/93576 objects misplaced (2.010%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:15 smithi114 ceph-mon[89482]: pgmap v254: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 7298/89430 objects degraded (8.161%); 1881/89430 objects misplaced (2.103%); 917 KiB/s, 14 objects/s recovering 2024-04-12T08:57:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:15 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/665370713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:15 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7298/89430 objects degraded (8.161%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:15 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:15 smithi090 ceph-mon[95814]: pgmap v254: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 7298/89430 objects degraded (8.161%); 1881/89430 objects misplaced (2.103%); 917 KiB/s, 14 objects/s recovering 2024-04-12T08:57:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:15 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/665370713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:15 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7298/89430 objects degraded (8.161%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:15 smithi090 ceph-mon[99575]: pgmap v254: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 7298/89430 objects degraded (8.161%); 1881/89430 objects misplaced (2.103%); 917 KiB/s, 14 objects/s recovering 2024-04-12T08:57:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:15 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/665370713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:15 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7298/89430 objects degraded (8.161%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:15 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:17 smithi114 ceph-mon[89482]: pgmap v255: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 7298/87978 objects degraded (8.295%); 1881/87978 objects misplaced (2.138%); 917 KiB/s, 14 objects/s recovering 2024-04-12T08:57:17.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:17 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4202515206' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:17 smithi090 ceph-mon[95814]: pgmap v255: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 7298/87978 objects degraded (8.295%); 1881/87978 objects misplaced (2.138%); 917 KiB/s, 14 objects/s recovering 2024-04-12T08:57:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:17 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4202515206' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:17.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:17 smithi090 ceph-mon[99575]: pgmap v255: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 7298/87978 objects degraded (8.295%); 1881/87978 objects misplaced (2.138%); 917 KiB/s, 14 objects/s recovering 2024-04-12T08:57:17.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:17 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4202515206' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:19.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:19 smithi114 ceph-mon[89482]: pgmap v256: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 7447/78033 objects degraded (9.543%); 1637/78033 objects misplaced (2.098%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:19 smithi090 ceph-mon[95814]: pgmap v256: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 7447/78033 objects degraded (9.543%); 1637/78033 objects misplaced (2.098%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:19.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:19 smithi090 ceph-mon[99575]: pgmap v256: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 7447/78033 objects degraded (9.543%); 1637/78033 objects misplaced (2.098%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T08:57:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:20 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3398201413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:20 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7447/78033 objects degraded (9.543%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:20 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3398201413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:20 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7447/78033 objects degraded (9.543%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:20 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3398201413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:20 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7447/78033 objects degraded (9.543%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:21.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:21 smithi090 ceph-mon[99575]: pgmap v257: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 7447/78033 objects degraded (9.543%); 1637/78033 objects misplaced (2.098%); 954 KiB/s, 14 objects/s recovering 2024-04-12T08:57:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:21 smithi090 ceph-mon[95814]: pgmap v257: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 7447/78033 objects degraded (9.543%); 1637/78033 objects misplaced (2.098%); 954 KiB/s, 14 objects/s recovering 2024-04-12T08:57:22.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:21 smithi114 ceph-mon[89482]: pgmap v257: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 7447/78033 objects degraded (9.543%); 1637/78033 objects misplaced (2.098%); 954 KiB/s, 14 objects/s recovering 2024-04-12T08:57:23.634 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:57:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:57:23] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.43.0" 2024-04-12T08:57:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:23 smithi090 ceph-mon[99575]: pgmap v258: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 7574/71067 objects degraded (10.658%); 1411/71067 objects misplaced (1.985%); 1.4 MiB/s, 23 objects/s recovering 2024-04-12T08:57:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:23 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3657720858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:23 smithi090 ceph-mon[95814]: pgmap v258: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 7574/71067 objects degraded (10.658%); 1411/71067 objects misplaced (1.985%); 1.4 MiB/s, 23 objects/s recovering 2024-04-12T08:57:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:23 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3657720858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:23 smithi114 ceph-mon[89482]: pgmap v258: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 7574/71067 objects degraded (10.658%); 1411/71067 objects misplaced (1.985%); 1.4 MiB/s, 23 objects/s recovering 2024-04-12T08:57:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:23 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3657720858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:25.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:25 smithi090 ceph-mon[99575]: pgmap v259: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 7574/66777 objects degraded (11.342%); 1411/66777 objects misplaced (2.113%); 1.0 MiB/s, 16 objects/s recovering 2024-04-12T08:57:25.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1478804543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:25.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:25 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7574/66777 objects degraded (11.342%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:25.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:25 smithi090 ceph-mon[95814]: pgmap v259: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 7574/66777 objects degraded (11.342%); 1411/66777 objects misplaced (2.113%); 1.0 MiB/s, 16 objects/s recovering 2024-04-12T08:57:25.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1478804543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:25.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:25 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7574/66777 objects degraded (11.342%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:25 smithi114 ceph-mon[89482]: pgmap v259: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 7574/66777 objects degraded (11.342%); 1411/66777 objects misplaced (2.113%); 1.0 MiB/s, 16 objects/s recovering 2024-04-12T08:57:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1478804543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:25 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7574/66777 objects degraded (11.342%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:26.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:26 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:57:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:26 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:57:27.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:26 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:57:27.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:27 smithi090 ceph-mon[99575]: pgmap v260: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 7574/65355 objects degraded (11.589%); 1411/65355 objects misplaced (2.159%); 1.0 MiB/s, 16 objects/s recovering 2024-04-12T08:57:27.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:27 smithi090 ceph-mon[95814]: pgmap v260: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 7574/65355 objects degraded (11.589%); 1411/65355 objects misplaced (2.159%); 1.0 MiB/s, 16 objects/s recovering 2024-04-12T08:57:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:27 smithi114 ceph-mon[89482]: pgmap v260: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 7574/65355 objects degraded (11.589%); 1411/65355 objects misplaced (2.159%); 1.0 MiB/s, 16 objects/s recovering 2024-04-12T08:57:28.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:28 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1911584326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:28.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:28 smithi090 ceph-mon[99575]: pgmap v261: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 7723/55791 objects degraded (13.843%); 1164/55791 objects misplaced (2.086%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T08:57:28.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:28 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1911584326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:28.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:28 smithi090 ceph-mon[95814]: pgmap v261: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 7723/55791 objects degraded (13.843%); 1164/55791 objects misplaced (2.086%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T08:57:29.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:28 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1911584326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:29.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:28 smithi114 ceph-mon[89482]: pgmap v261: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 7723/55791 objects degraded (13.843%); 1164/55791 objects misplaced (2.086%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T08:57:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:31 smithi114 ceph-mon[89482]: pgmap v262: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 7723/55791 objects degraded (13.843%); 1164/55791 objects misplaced (2.086%); 1.0 MiB/s, 16 objects/s recovering 2024-04-12T08:57:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:31 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7723/55791 objects degraded (13.843%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:31 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/466628672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:31.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:31 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:31.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:31 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[99575]: pgmap v262: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 7723/55791 objects degraded (13.843%); 1164/55791 objects misplaced (2.086%); 1.0 MiB/s, 16 objects/s recovering 2024-04-12T08:57:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7723/55791 objects degraded (13.843%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/466628672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[95814]: pgmap v262: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 7723/55791 objects degraded (13.843%); 1164/55791 objects misplaced (2.086%); 1.0 MiB/s, 16 objects/s recovering 2024-04-12T08:57:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7723/55791 objects degraded (13.843%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/466628672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:31.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:31 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:33.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:33 smithi114 ceph-mon[89482]: pgmap v263: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.0 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 7850/49605 objects degraded (15.825%); 945/49605 objects misplaced (1.905%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T08:57:33.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:57:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:57:33] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.43.0" 2024-04-12T08:57:33.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:33 smithi090 ceph-mon[95814]: pgmap v263: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.0 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 7850/49605 objects degraded (15.825%); 945/49605 objects misplaced (1.905%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T08:57:33.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:33 smithi090 ceph-mon[99575]: pgmap v263: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.0 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 7850/49605 objects degraded (15.825%); 945/49605 objects misplaced (1.905%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T08:57:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:34 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3731080676' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:34 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:34 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:34.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:34 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3731080676' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:34.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:34 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:34.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:34 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:34.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:34 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3731080676' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:34.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:34 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:34.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:34 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:34.922 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:57:34 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:57:34.504+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:35.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: pgmap v264: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 956 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 7850/45933 objects degraded (17.090%); 945/45933 objects misplaced (2.057%); 1013 KiB/s, 15 objects/s recovering 2024-04-12T08:57:35.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:57:35.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:57:35.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: pgmap v265: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 956 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 7850/45933 objects degraded (17.090%); 945/45933 objects misplaced (2.057%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T08:57:35.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:35.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:57:35.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:57:35.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:57:35.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:57:35.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:35.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:35.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:35 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:35.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: pgmap v264: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 956 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 7850/45933 objects degraded (17.090%); 945/45933 objects misplaced (2.057%); 1013 KiB/s, 15 objects/s recovering 2024-04-12T08:57:35.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:57:35.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:57:35.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: pgmap v265: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 956 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 7850/45933 objects degraded (17.090%); 945/45933 objects misplaced (2.057%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T08:57:35.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: pgmap v264: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 956 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 7850/45933 objects degraded (17.090%); 945/45933 objects misplaced (2.057%); 1013 KiB/s, 15 objects/s recovering 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: pgmap v265: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 956 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 7850/45933 objects degraded (17.090%); 945/45933 objects misplaced (2.057%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:57:35.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:57:35.425 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:57:35.425 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:57:35.425 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:57:35.425 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:35.425 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:35.425 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:35 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:36.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:36 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7850/45933 objects degraded (17.090%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:36.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:36 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1798992371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:36.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:36 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7850/45933 objects degraded (17.090%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:36.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:36 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1798992371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:36.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:36 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7850/45933 objects degraded (17.090%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:36.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:36 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1798992371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:37.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:37 smithi114 ceph-mon[89482]: pgmap v266: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 928 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 7850/44610 objects degraded (17.597%); 945/44610 objects misplaced (2.118%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T08:57:37.422 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:37 smithi090 ceph-mon[95814]: pgmap v266: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 928 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 7850/44610 objects degraded (17.597%); 945/44610 objects misplaced (2.118%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T08:57:37.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:37 smithi090 ceph-mon[99575]: pgmap v266: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 928 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 7850/44610 objects degraded (17.597%); 945/44610 objects misplaced (2.118%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T08:57:38.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:38 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1613205553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:38.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:38 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1613205553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:38.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:38 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1613205553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:39.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:39 smithi090 ceph-mon[95814]: pgmap v267: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 734 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 7939/35271 objects degraded (22.509%); 740/35271 objects misplaced (2.098%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:57:39.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:39 smithi090 ceph-mon[99575]: pgmap v267: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 734 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 7939/35271 objects degraded (22.509%); 740/35271 objects misplaced (2.098%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:57:39.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:39 smithi114 ceph-mon[89482]: pgmap v267: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 734 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 7939/35271 objects degraded (22.509%); 740/35271 objects misplaced (2.098%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:57:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:41 smithi090 ceph-mon[95814]: pgmap v268: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 734 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 7939/35271 objects degraded (22.509%); 740/35271 objects misplaced (2.098%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:57:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:41 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7939/35271 objects degraded (22.509%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:41 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/768573886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:41.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:41 smithi090 ceph-mon[99575]: pgmap v268: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 734 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 7939/35271 objects degraded (22.509%); 740/35271 objects misplaced (2.098%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:57:41.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:41 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7939/35271 objects degraded (22.509%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:41.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:41 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/768573886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:41.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:41 smithi114 ceph-mon[89482]: pgmap v268: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 734 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 7939/35271 objects degraded (22.509%); 740/35271 objects misplaced (2.098%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:57:41.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:41 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7939/35271 objects degraded (22.509%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:41.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:41 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/768573886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:43 smithi090 ceph-mon[95814]: pgmap v269: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 559 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 8048/26886 objects degraded (29.934%); 522/26886 objects misplaced (1.942%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T08:57:43.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:57:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:57:43] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.43.0" 2024-04-12T08:57:43.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:43 smithi090 ceph-mon[99575]: pgmap v269: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 559 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 8048/26886 objects degraded (29.934%); 522/26886 objects misplaced (1.942%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T08:57:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:43 smithi114 ceph-mon[89482]: pgmap v269: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 559 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 8048/26886 objects degraded (29.934%); 522/26886 objects misplaced (1.942%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T08:57:44.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:44 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/860318486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:44.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:44 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/860318486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:44 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/860318486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:45.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:45 smithi090 ceph-mon[99575]: pgmap v270: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 501 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 8048/24102 objects degraded (33.391%); 522/24102 objects misplaced (2.166%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T08:57:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:45 smithi090 ceph-mon[95814]: pgmap v270: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 501 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 8048/24102 objects degraded (33.391%); 522/24102 objects misplaced (2.166%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T08:57:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:45 smithi114 ceph-mon[89482]: pgmap v270: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 501 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 8048/24102 objects degraded (33.391%); 522/24102 objects misplaced (2.166%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T08:57:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:46 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 8048/24102 objects degraded (33.391%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:46 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:46 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3780306137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:46.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:46 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 8048/24102 objects degraded (33.391%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:46.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:46 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:46.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:46 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3780306137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:46 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 8048/24102 objects degraded (33.391%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:46 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:57:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:46 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3780306137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:47.767 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:47 smithi114 ceph-mon[89482]: pgmap v271: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 471 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 8048/22674 objects degraded (35.494%); 522/22674 objects misplaced (2.302%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T08:57:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:47 smithi090 ceph-mon[95814]: pgmap v271: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 471 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 8048/22674 objects degraded (35.494%); 522/22674 objects misplaced (2.302%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T08:57:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:47 smithi090 ceph-mon[99575]: pgmap v271: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 471 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 8048/22674 objects degraded (35.494%); 522/22674 objects misplaced (2.302%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T08:57:49.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:49 smithi114 ceph-mon[89482]: pgmap v272: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 268 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 8193/12924 objects degraded (63.394%); 276/12924 objects misplaced (2.136%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T08:57:49.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:49 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2755787903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:49.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:57:49 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:57:49.522+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:49 smithi090 ceph-mon[95814]: pgmap v272: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 268 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 8193/12924 objects degraded (63.394%); 276/12924 objects misplaced (2.136%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T08:57:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:49 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2755787903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:49 smithi090 ceph-mon[99575]: pgmap v272: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 268 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 8193/12924 objects degraded (63.394%); 276/12924 objects misplaced (2.136%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T08:57:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:49 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2755787903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:50 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:50 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:50 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:50 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 8193/12924 objects degraded (63.394%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:50 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:50 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:50 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:50 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 8193/12924 objects degraded (63.394%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:50 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:50 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:57:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:50 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T08:57:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:50 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 8193/12924 objects degraded (63.394%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:51.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:51 smithi114 ceph-mon[89482]: pgmap v273: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 268 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 8193/12924 objects degraded (63.394%); 276/12924 objects misplaced (2.136%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:57:51.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:51 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2278439815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:51 smithi090 ceph-mon[95814]: pgmap v273: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 268 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 8193/12924 objects degraded (63.394%); 276/12924 objects misplaced (2.136%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:57:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:51 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2278439815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:51 smithi090 ceph-mon[99575]: pgmap v273: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 268 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 8193/12924 objects degraded (63.394%); 276/12924 objects misplaced (2.136%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:57:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:51 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2278439815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:52.613 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Removed 122420 objects 2024-04-12T08:57:52.613 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Clean up completed and total clean up time :167.456 2024-04-12T08:57:53.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:53 smithi090 ceph-mon[99575]: pgmap v274: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 98 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 8304/4734 objects degraded (175.412%); 54/4734 objects misplaced (1.141%); 1.7 MiB/s, 26 objects/s recovering 2024-04-12T08:57:53.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:53 smithi090 ceph-mon[95814]: pgmap v274: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 98 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 8304/4734 objects degraded (175.412%); 54/4734 objects misplaced (1.141%); 1.7 MiB/s, 26 objects/s recovering 2024-04-12T08:57:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:57:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:57:53] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2024-04-12T08:57:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:53 smithi114 ceph-mon[89482]: pgmap v274: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 98 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 8304/4734 objects degraded (175.412%); 54/4734 objects misplaced (1.141%); 1.7 MiB/s, 26 objects/s recovering 2024-04-12T08:57:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:54 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2797801033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:54.911 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:54 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2797801033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:54.911 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:54 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2797801033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:55.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:55 smithi114 ceph-mon[89482]: pgmap v275: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 39 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 8304/1929 objects degraded (430.482%); 54/1929 objects misplaced (2.799%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:57:55.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:55 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 8304/1929 objects degraded (430.482%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:55 smithi090 ceph-mon[99575]: pgmap v275: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 39 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 8304/1929 objects degraded (430.482%); 54/1929 objects misplaced (2.799%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:57:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:55 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 8304/1929 objects degraded (430.482%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:55 smithi090 ceph-mon[95814]: pgmap v275: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 39 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 8304/1929 objects degraded (430.482%); 54/1929 objects misplaced (2.799%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:57:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:55 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 8304/1929 objects degraded (430.482%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:57:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:56 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1871334153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:56.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:56 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1871334153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:56 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1871334153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:57.466 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-04-12T08:57:57.467 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-04-12T08:57:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:57:57.854 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:57 smithi090 ceph-mon[99575]: pgmap v276: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 28 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8304/1380 objects degraded (601.739%); 54/1380 objects misplaced (3.913%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:57:57.854 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:57 smithi090 ceph-mon[95814]: pgmap v276: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 28 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8304/1380 objects degraded (601.739%); 54/1380 objects misplaced (3.913%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:57:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:57 smithi114 ceph-mon[89482]: pgmap v276: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 28 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8304/1380 objects degraded (601.739%); 54/1380 objects misplaced (3.913%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T08:57:58.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:58 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2816578927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:58.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:58 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2816578927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:58 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2816578927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:57:59.756 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:57:59 smithi090 ceph-mon[99575]: pgmap v277: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 8336/72 objects degraded (11577.778%); 1.2 MiB/s, 49 objects/s recovering 2024-04-12T08:57:59.756 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:57:59 smithi090 ceph-mon[95814]: pgmap v277: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 8336/72 objects degraded (11577.778%); 1.2 MiB/s, 49 objects/s recovering 2024-04-12T08:57:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:57:59 smithi114 ceph-mon[89482]: pgmap v277: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 8336/72 objects degraded (11577.778%); 1.2 MiB/s, 49 objects/s recovering 2024-04-12T08:58:00.435 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'unique_pool_2' removed 2024-04-12T08:58:00.788 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:00 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1612885592' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:58:00.788 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:00 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 8336/72 objects degraded (11577.778%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:58:00.788 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:00 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1612885592' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T08:58:00.788 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:00 smithi090 ceph-mon[99575]: osdmap e108: 8 total, 8 up, 8 in 2024-04-12T08:58:00.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:00 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1612885592' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:58:00.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:00 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 8336/72 objects degraded (11577.778%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:58:00.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:00 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1612885592' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T08:58:00.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:00 smithi090 ceph-mon[95814]: osdmap e108: 8 total, 8 up, 8 in 2024-04-12T08:58:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:00 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1612885592' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T08:58:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:00 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 8336/72 objects degraded (11577.778%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T08:58:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:00 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1612885592' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T08:58:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:00 smithi114 ceph-mon[89482]: osdmap e108: 8 total, 8 up, 8 in 2024-04-12T08:58:01.144 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-12T08:58:01.144 INFO:tasks.radosbench:Beginning radosbench... 2024-04-12T08:58:01.144 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-04-12T08:58:01.145 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-04-12T08:58:01.402 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:58:01.776 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:01 smithi090 ceph-mon[99575]: pgmap v278: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 8336/72 objects degraded (11577.778%); 709 KiB/s, 41 objects/s recovering 2024-04-12T08:58:01.776 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:01.776 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:01 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2514193860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:01.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:01 smithi090 ceph-mon[95814]: pgmap v278: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 8336/72 objects degraded (11577.778%); 709 KiB/s, 41 objects/s recovering 2024-04-12T08:58:01.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:01.776 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:01 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2514193860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:01 smithi114 ceph-mon[89482]: pgmap v278: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 8336/72 objects degraded (11577.778%); 709 KiB/s, 41 objects/s recovering 2024-04-12T08:58:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:01 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:01 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2514193860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:02.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:02 smithi090 ceph-mon[99575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8336/72 objects degraded (11577.778%), 4 pgs degraded, 4 pgs undersized) 2024-04-12T08:58:02.714 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:02 smithi090 ceph-mon[95814]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8336/72 objects degraded (11577.778%), 4 pgs degraded, 4 pgs undersized) 2024-04-12T08:58:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:02 smithi114 ceph-mon[89482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8336/72 objects degraded (11577.778%), 4 pgs degraded, 4 pgs undersized) 2024-04-12T08:58:03.645 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:58:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:58:03] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.43.0" 2024-04-12T08:58:03.699 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'unique_pool_3' created 2024-04-12T08:58:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:03 smithi090 ceph-mon[99575]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:03 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3972072326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:03.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:03 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/74192234' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-12T08:58:03.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:03 smithi090 ceph-mon[95814]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:03.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:03 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3972072326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:03.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:03 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/74192234' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-12T08:58:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:03 smithi114 ceph-mon[89482]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:03 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3972072326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:04.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:03 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/74192234' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-12T08:58:04.452 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-04-12T08:58:04.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:58:04.538+0000 7efe43a91700 -1 mgr.server reply reply (11) Resource temporarily unavailable 16 pgs have unknown state; cannot draw any conclusions 2024-04-12T08:58:04.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:58:04.538+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy 16 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2024-04-12T08:58:04.723 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T08:58:05.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/74192234' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-12T08:58:05.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[95814]: osdmap e109: 8 total, 8 up, 8 in 2024-04-12T08:58:05.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[95814]: pgmap v282: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:05.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:05.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:05.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[95814]: Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2024-04-12T08:58:05.040 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/74192234' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-12T08:58:05.041 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[99575]: osdmap e109: 8 total, 8 up, 8 in 2024-04-12T08:58:05.041 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[99575]: pgmap v282: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:05.041 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:05.041 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:05.041 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:04 smithi090 ceph-mon[99575]: Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2024-04-12T08:58:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:04 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/74192234' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-12T08:58:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:04 smithi114 ceph-mon[89482]: osdmap e109: 8 total, 8 up, 8 in 2024-04-12T08:58:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:04 smithi114 ceph-mon[89482]: pgmap v282: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:04 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:04 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:05.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:04 smithi114 ceph-mon[89482]: Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2024-04-12T08:58:05.983 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:05 smithi090 ceph-mon[95814]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:58:05.983 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:05 smithi090 ceph-mon[95814]: osdmap e110: 8 total, 8 up, 8 in 2024-04-12T08:58:05.983 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:05 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1027247840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:05.983 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:05 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:05.983 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:05 smithi090 ceph-mon[99575]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:58:05.983 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:05 smithi090 ceph-mon[99575]: osdmap e110: 8 total, 8 up, 8 in 2024-04-12T08:58:05.983 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:05 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1027247840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:05.983 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:05 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:06.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:05 smithi114 ceph-mon[89482]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T08:58:06.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:05 smithi114 ceph-mon[89482]: osdmap e110: 8 total, 8 up, 8 in 2024-04-12T08:58:06.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:05 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1027247840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:06.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:05 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:06.727 INFO:teuthology.orchestra.run.smithi090.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-04-12T08:58:07.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:06 smithi114 ceph-mon[89482]: osdmap e111: 8 total, 8 up, 8 in 2024-04-12T08:58:07.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:06 smithi114 ceph-mon[89482]: pgmap v285: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:07.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:06 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1105711517' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:58:07.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:06 smithi114 ceph-mon[89482]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:58:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:06 smithi090 ceph-mon[95814]: osdmap e111: 8 total, 8 up, 8 in 2024-04-12T08:58:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:06 smithi090 ceph-mon[95814]: pgmap v285: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:06 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1105711517' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:58:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:06 smithi090 ceph-mon[95814]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:58:07.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:06 smithi090 ceph-mon[99575]: osdmap e111: 8 total, 8 up, 8 in 2024-04-12T08:58:07.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:06 smithi090 ceph-mon[99575]: pgmap v285: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:07.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:06 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1105711517' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:58:07.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:06 smithi090 ceph-mon[99575]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T08:58:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:07 smithi114 ceph-mon[89482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:58:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:07 smithi114 ceph-mon[89482]: osdmap e112: 8 total, 8 up, 8 in 2024-04-12T08:58:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:07 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3259541803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:08.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:07 smithi090 ceph-mon[99575]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:58:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:07 smithi090 ceph-mon[99575]: osdmap e112: 8 total, 8 up, 8 in 2024-04-12T08:58:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:07 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3259541803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:07 smithi090 ceph-mon[95814]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T08:58:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:07 smithi090 ceph-mon[95814]: osdmap e112: 8 total, 8 up, 8 in 2024-04-12T08:58:08.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:07 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3259541803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:08.820 DEBUG:teuthology.orchestra.run.smithi090:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-04-12T08:58:08.822 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-12T08:58:08.823 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-12T08:58:08.871 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:hints = 1 2024-04-12T08:58:08.871 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-12T08:58:08.871 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Object prefix: benchmark_data_smithi090_115603 2024-04-12T08:58:08.871 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:58:08.871 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 0 0 0 0 0 0 - 0 2024-04-12T08:58:09.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:08 smithi114 ceph-mon[89482]: pgmap v287: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:09.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:08 smithi090 ceph-mon[99575]: pgmap v287: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:09.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:08 smithi090 ceph-mon[95814]: pgmap v287: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:09.871 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 1 16 3978 3962 247.61 247.625 0.00436263 0.0040241 2024-04-12T08:58:10.111 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:09 smithi090 ceph-mon[99575]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:58:10.111 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:09 smithi090 ceph-mon[95814]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:58:10.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:09 smithi114 ceph-mon[89482]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T08:58:10.871 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 2 16 7629 7613 237.885 228.188 0.00171082 0.0041931 2024-04-12T08:58:11.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:10 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2845326729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:11.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:10 smithi114 ceph-mon[89482]: pgmap v288: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:11.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:10 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2845326729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:11.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:10 smithi090 ceph-mon[99575]: pgmap v288: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:11.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:10 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2845326729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:11.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:10 smithi090 ceph-mon[95814]: pgmap v288: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T08:58:11.871 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 3 16 11193 11177 232.834 222.75 0.00666592 0.00428701 2024-04-12T08:58:12.871 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 4 16 14136 14120 220.606 183.938 0.00940441 0.00452605 2024-04-12T08:58:13.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:13 smithi114 ceph-mon[89482]: pgmap v289: 89 pgs: 89 active+clean; 488 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T08:58:13.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:13 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1842837255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:58:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:58:13] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.43.0" 2024-04-12T08:58:13.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:13 smithi090 ceph-mon[99575]: pgmap v289: 89 pgs: 89 active+clean; 488 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T08:58:13.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:13 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1842837255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:13.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:13 smithi090 ceph-mon[95814]: pgmap v289: 89 pgs: 89 active+clean; 488 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T08:58:13.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:13 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1842837255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:13.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 5 16 17290 17274 215.906 197.125 0.0059285 0.00462515 2024-04-12T08:58:14.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 6 16 20239 20223 210.638 184.312 0.00292954 0.00474132 2024-04-12T08:58:15.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:15 smithi114 ceph-mon[89482]: pgmap v290: 89 pgs: 89 active+clean; 643 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-04-12T08:58:15.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:15 smithi090 ceph-mon[99575]: pgmap v290: 89 pgs: 89 active+clean; 643 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-04-12T08:58:15.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:15 smithi090 ceph-mon[95814]: pgmap v290: 89 pgs: 89 active+clean; 643 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-04-12T08:58:15.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 7 16 22698 22682 202.5 153.688 0.00439525 0.00493312 2024-04-12T08:58:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:16 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4137681816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:16 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:16 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:16 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:16.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:16 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4137681816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:16.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:16 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:16.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:16.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:16 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:16.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:16 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4137681816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:16.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:16 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:16.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:16 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:16.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:16 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:16.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 8 16 25415 25399 198.412 169.812 0.00796996 0.00503417 2024-04-12T08:58:17.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 9 16 27851 27835 193.282 152.25 0.00643972 0.00516775 2024-04-12T08:58:17.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:17 smithi090 ceph-mon[99575]: pgmap v291: 89 pgs: 89 active+clean; 826 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-04-12T08:58:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:17 smithi090 ceph-mon[95814]: pgmap v291: 89 pgs: 89 active+clean; 826 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-04-12T08:58:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:17 smithi114 ceph-mon[89482]: pgmap v291: 89 pgs: 89 active+clean; 826 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-04-12T08:58:18.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 10 16 30256 30240 188.983 150.312 0.00646855 0.00528601 2024-04-12T08:58:18.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:18 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3328730901' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:18.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:18 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3328730901' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:19.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:18 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3328730901' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:19.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 11 16 32404 32388 184.006 134.25 0.00803312 0.00542941 2024-04-12T08:58:19.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:19 smithi090 ceph-mon[99575]: pgmap v292: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-04-12T08:58:19.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:19 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:19 smithi090 ceph-mon[95814]: pgmap v292: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-04-12T08:58:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:19 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:20.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:19 smithi114 ceph-mon[89482]: pgmap v292: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-04-12T08:58:20.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:19 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:20.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 12 16 34342 34326 178.765 121.125 0.00799309 0.00558896 2024-04-12T08:58:20.879 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:20 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:20.879 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:20 smithi090 ceph-mon[99575]: Upgrade: osd.1 is safe to restart 2024-04-12T08:58:20.880 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:20 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:20.880 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:20 smithi090 ceph-mon[95814]: Upgrade: osd.1 is safe to restart 2024-04-12T08:58:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:20 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T08:58:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:20 smithi114 ceph-mon[89482]: Upgrade: osd.1 is safe to restart 2024-04-12T08:58:21.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 13 16 36206 36190 173.975 116.5 0.00489193 0.00574255 2024-04-12T08:58:21.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:21 smithi090 ceph-mon[95814]: pgmap v293: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-04-12T08:58:21.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:21 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1924646430' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:21 smithi090 ceph-mon[99575]: pgmap v293: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-04-12T08:58:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:21 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1924646430' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:22.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:21 smithi114 ceph-mon[89482]: pgmap v293: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-04-12T08:58:22.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:21 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1924646430' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:22.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 14 16 38221 38205 170.543 125.938 0.00483808 0.00585814 2024-04-12T08:58:23.651 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:58:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:58:23] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-04-12T08:58:23.872 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 15 16 39629 39613 165.039 88 0.00303362 0.00603832 2024-04-12T08:58:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:23 smithi090 ceph-mon[99575]: pgmap v294: 89 pgs: 89 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 176 MiB/s wr, 2.81k op/s 2024-04-12T08:58:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:23 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2261000596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:23 smithi090 ceph-mon[95814]: pgmap v294: 89 pgs: 89 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 176 MiB/s wr, 2.81k op/s 2024-04-12T08:58:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:23 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2261000596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:23 smithi114 ceph-mon[89482]: pgmap v294: 89 pgs: 89 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 176 MiB/s wr, 2.81k op/s 2024-04-12T08:58:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:23 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2261000596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:24.873 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 16 15 41368 41353 161.52 108.75 0.0101389 0.00618604 2024-04-12T08:58:25.873 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 17 16 42518 42502 156.243 71.8125 0.00496014 0.00638801 2024-04-12T08:58:25.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:25 smithi090 ceph-mon[99575]: pgmap v295: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-04-12T08:58:25.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/351866449' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:25.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:25 smithi090 ceph-mon[95814]: pgmap v295: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-04-12T08:58:25.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/351866449' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:25 smithi114 ceph-mon[89482]: pgmap v295: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-04-12T08:58:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/351866449' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:26.873 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 18 16 44215 44199 153.455 106.062 0.0116775 0.00651244 2024-04-12T08:58:27.873 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 19 16 45987 45971 151.206 110.75 0.0122072 0.00660806 2024-04-12T08:58:27.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:27 smithi090 ceph-mon[95814]: pgmap v296: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-04-12T08:58:27.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:27 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:27.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:27 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T08:58:27.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:27 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:58:27.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:27 smithi090 ceph-mon[99575]: pgmap v296: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-04-12T08:58:27.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:27 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:27.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:27 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T08:58:27.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:27 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:58:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:27 smithi114 ceph-mon[89482]: pgmap v296: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-04-12T08:58:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:27 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:27 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T08:58:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:27 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:58:28.664 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:28 smithi090 ceph-mon[95814]: Upgrade: Updating osd.1 2024-04-12T08:58:28.664 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:28 smithi090 ceph-mon[99575]: Upgrade: Updating osd.1 2024-04-12T08:58:28.664 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:28 smithi090 ceph-mon[99575]: Deploying daemon osd.1 on smithi090 2024-04-12T08:58:28.664 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:28 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2063709522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:28.665 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:28 smithi090 ceph-mon[99575]: pgmap v297: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 159 MiB/s wr, 2.55k op/s 2024-04-12T08:58:28.873 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:58:28.873611+0000 min lat: 0.00116602 max lat: 0.16379 avg lat: 0.00670788 2024-04-12T08:58:28.873 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:58:28.873 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 20 16 47686 47670 148.955 106.188 0.00699956 0.00670788 2024-04-12T08:58:28.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:28 smithi090 ceph-mon[95814]: Deploying daemon osd.1 on smithi090 2024-04-12T08:58:28.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:28 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2063709522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:28.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:28 smithi090 ceph-mon[95814]: pgmap v297: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 159 MiB/s wr, 2.55k op/s 2024-04-12T08:58:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:28 smithi114 ceph-mon[89482]: Upgrade: Updating osd.1 2024-04-12T08:58:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:28 smithi114 ceph-mon[89482]: Deploying daemon osd.1 on smithi090 2024-04-12T08:58:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:28 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2063709522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:29.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:28 smithi114 ceph-mon[89482]: pgmap v297: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 159 MiB/s wr, 2.55k op/s 2024-04-12T08:58:29.873 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 21 16 49407 49391 146.983 107.562 0.00444018 0.0067983 2024-04-12T08:58:30.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 22 16 50949 50933 144.682 96.375 0.0121074 0.0069063 2024-04-12T08:58:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:31 smithi114 ceph-mon[89482]: pgmap v298: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-04-12T08:58:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:31 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1055319862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:31 smithi090 ceph-mon[95814]: pgmap v298: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-04-12T08:58:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:31 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1055319862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:31 smithi090 ceph-mon[99575]: pgmap v298: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-04-12T08:58:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:31 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1055319862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:31.873 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 23 16 52689 52673 143.12 108.75 0.00787164 0.00698227 2024-04-12T08:58:32.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 24 16 54267 54251 141.265 98.625 0.0104836 0.00707409 2024-04-12T08:58:32.922 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:32 smithi090 systemd[1]: Stopping Ceph osd.1 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:58:33.173 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:32 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[49872]: 2024-04-12T08:58:32.925+0000 7fa20942e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T08:58:33.173 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:32 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[49872]: 2024-04-12T08:58:32.925+0000 7fa20942e700 -1 osd.1 112 *** Got signal Terminated *** 2024-04-12T08:58:33.173 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:32 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[49872]: 2024-04-12T08:58:32.925+0000 7fa20942e700 -1 osd.1 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T08:58:33.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:33 smithi114 ceph-mon[89482]: pgmap v299: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-04-12T08:58:33.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:33 smithi114 ceph-mon[89482]: osd.1 marked itself down and dead 2024-04-12T08:58:33.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:58:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:58:33] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2024-04-12T08:58:33.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:33 smithi090 ceph-mon[95814]: pgmap v299: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-04-12T08:58:33.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:33 smithi090 ceph-mon[95814]: osd.1 marked itself down and dead 2024-04-12T08:58:33.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:33 smithi090 ceph-mon[99575]: pgmap v299: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-04-12T08:58:33.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:33 smithi090 ceph-mon[99575]: osd.1 marked itself down and dead 2024-04-12T08:58:33.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 25 16 55075 55059 137.635 50.5 0.00188231 0.00712329 2024-04-12T08:58:34.363 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:33 smithi090 podman[116232]: 2024-04-12 08:58:33.985150947 +0000 UTC m=+1.243039831 container died 98eb8b5854e94ca1d125d6c46479b8e965eb062ceeea9445267a72e22eda47b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-12T08:58:34.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:34 smithi114 ceph-mon[89482]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T08:58:34.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:34 smithi114 ceph-mon[89482]: osdmap e113: 8 total, 7 up, 8 in 2024-04-12T08:58:34.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:34 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2663522409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:34.673 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:34 smithi090 podman[116232]: 2024-04-12 08:58:34.46652588 +0000 UTC m=+1.724414769 container cleanup 98eb8b5854e94ca1d125d6c46479b8e965eb062ceeea9445267a72e22eda47b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1, RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-12T08:58:34.673 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:34 smithi090 bash[116232]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1 2024-04-12T08:58:34.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:34 smithi090 ceph-mon[95814]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T08:58:34.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:34 smithi090 ceph-mon[95814]: osdmap e113: 8 total, 7 up, 8 in 2024-04-12T08:58:34.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:34 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2663522409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:34.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:34 smithi090 ceph-mon[99575]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T08:58:34.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:34 smithi090 ceph-mon[99575]: osdmap e113: 8 total, 7 up, 8 in 2024-04-12T08:58:34.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:34 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2663522409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:34.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 26 16 55986 55970 134.531 56.9375 0.0115043 0.00742781 2024-04-12T08:58:35.173 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:34 smithi090 podman[116271]: 2024-04-12 08:58:34.762165361 +0000 UTC m=+0.775493625 container remove 98eb8b5854e94ca1d125d6c46479b8e965eb062ceeea9445267a72e22eda47b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, ceph=True, GIT_CLEAN=True) 2024-04-12T08:58:35.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:35 smithi090 ceph-mon[99575]: pgmap v301: 89 pgs: 13 stale+active+clean, 76 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-04-12T08:58:35.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:35 smithi090 ceph-mon[99575]: osdmap e114: 8 total, 7 up, 8 in 2024-04-12T08:58:35.673 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:35 smithi090 podman[116351]: 2024-04-12 08:58:35.153262436 +0000 UTC m=+0.028687626 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:58:35.673 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:35 smithi090 podman[116351]: 2024-04-12 08:58:35.272167095 +0000 UTC m=+0.147592268 container create e49b745c76c91c7b3db8638c68243b4f4f90711d399e059f1e320d2fa54e0ddb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T08:58:35.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:35 smithi090 ceph-mon[95814]: pgmap v301: 89 pgs: 13 stale+active+clean, 76 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-04-12T08:58:35.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:35 smithi090 ceph-mon[95814]: osdmap e114: 8 total, 7 up, 8 in 2024-04-12T08:58:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:35 smithi114 ceph-mon[89482]: pgmap v301: 89 pgs: 13 stale+active+clean, 76 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-04-12T08:58:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:35 smithi114 ceph-mon[89482]: osdmap e114: 8 total, 7 up, 8 in 2024-04-12T08:58:35.873 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 27 16 57747 57731 133.624 110.062 0.0107316 0.0074788 2024-04-12T08:58:36.173 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:35 smithi090 podman[116351]: 2024-04-12 08:58:35.843575977 +0000 UTC m=+0.719001154 container init e49b745c76c91c7b3db8638c68243b4f4f90711d399e059f1e320d2fa54e0ddb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS) 2024-04-12T08:58:36.173 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:35 smithi090 podman[116351]: 2024-04-12 08:58:35.8526781 +0000 UTC m=+0.728103272 container start e49b745c76c91c7b3db8638c68243b4f4f90711d399e059f1e320d2fa54e0ddb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0) 2024-04-12T08:58:36.173 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:35 smithi090 podman[116351]: 2024-04-12 08:58:35.962111481 +0000 UTC m=+0.837536650 container attach e49b745c76c91c7b3db8638c68243b4f4f90711d399e059f1e320d2fa54e0ddb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS) 2024-04-12T08:58:36.659 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:36 smithi090 conmon[116366]: conmon e49b745c76c91c7b3db8 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-e49b745c76c91c7b3db8638c68243b4f4f90711d399e059f1e320d2fa54e0ddb.scope/container/memory.events 2024-04-12T08:58:36.659 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:36 smithi090 podman[116351]: 2024-04-12 08:58:36.329911658 +0000 UTC m=+1.205336829 container died e49b745c76c91c7b3db8638c68243b4f4f90711d399e059f1e320d2fa54e0ddb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T08:58:36.659 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:36 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1740903687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:36.660 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:36 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1740903687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:36 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1740903687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:36.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 28 16 59638 59622 133.072 118.188 0.00649642 0.00750981 2024-04-12T08:58:37.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:37 smithi090 podman[116403]: 2024-04-12 08:58:37.102561457 +0000 UTC m=+0.763072260 container remove e49b745c76c91c7b3db8638c68243b4f4f90711d399e059f1e320d2fa54e0ddb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb) 2024-04-12T08:58:37.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:37 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-12T08:58:37.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:37 smithi090 systemd[1]: Stopped Ceph osd.1 for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:58:37.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:37 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.1.service: Consumed 17min 12.404s CPU time. 2024-04-12T08:58:37.718 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:37 smithi090 ceph-mon[95814]: pgmap v303: 89 pgs: 3 active+undersized+degraded, 10 stale+active+clean, 4 active+undersized, 72 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 3664/161727 objects degraded (2.266%) 2024-04-12T08:58:37.718 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:37 smithi090 ceph-mon[95814]: Health check failed: Degraded data redundancy: 3664/161727 objects degraded (2.266%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:37.719 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:37 smithi090 ceph-mon[99575]: pgmap v303: 89 pgs: 3 active+undersized+degraded, 10 stale+active+clean, 4 active+undersized, 72 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 3664/161727 objects degraded (2.266%) 2024-04-12T08:58:37.719 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:37 smithi090 ceph-mon[99575]: Health check failed: Degraded data redundancy: 3664/161727 objects degraded (2.266%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:37.719 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:37 smithi090 systemd[1]: Starting Ceph osd.1 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T08:58:37.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:37 smithi114 ceph-mon[89482]: pgmap v303: 89 pgs: 3 active+undersized+degraded, 10 stale+active+clean, 4 active+undersized, 72 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 3664/161727 objects degraded (2.266%) 2024-04-12T08:58:37.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:37 smithi114 ceph-mon[89482]: Health check failed: Degraded data redundancy: 3664/161727 objects degraded (2.266%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:37.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 29 16 61472 61456 132.436 114.625 0.00577492 0.00754569 2024-04-12T08:58:38.001 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:37 smithi090 podman[116492]: 2024-04-12 08:58:37.623584538 +0000 UTC m=+0.025695751 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:58:38.001 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:37 smithi090 podman[116492]: 2024-04-12 08:58:37.755528224 +0000 UTC m=+0.157639425 container create fec1006396f509075f84552bc3313fbcbb7b0563093929b75278bfa46d876d97 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS) 2024-04-12T08:58:38.001 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 podman[116492]: 2024-04-12 08:58:38.001274658 +0000 UTC m=+0.403385863 container init fec1006396f509075f84552bc3313fbcbb7b0563093929b75278bfa46d876d97 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-12T08:58:38.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 podman[116492]: 2024-04-12 08:58:38.007111536 +0000 UTC m=+0.409222739 container start fec1006396f509075f84552bc3313fbcbb7b0563093929b75278bfa46d876d97 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate, RELEASE=squid-b277feb, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T08:58:38.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 podman[116492]: 2024-04-12 08:58:38.10973309 +0000 UTC m=+0.511844292 container attach fec1006396f509075f84552bc3313fbcbb7b0563093929b75278bfa46d876d97 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T08:58:38.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 30 16 63038 63022 131.284 97.875 0.00472078 0.0076124 2024-04-12T08:58:39.173 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate[116507]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T08:58:39.173 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate[116507]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T08:58:39.173 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 bash[116492]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 bash[116492]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 bash[116492]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 bash[116492]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 bash[116492]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 bash[116492]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 bash[116492]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate[116507]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate[116507]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate[116507]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate[116507]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate[116507]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 conmon[116507]: conmon fec1006396f509075f84 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-fec1006396f509075f84552bc3313fbcbb7b0563093929b75278bfa46d876d97.scope/container/memory.events 2024-04-12T08:58:39.174 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:38 smithi090 podman[116666]: 2024-04-12 08:58:38.879762424 +0000 UTC m=+0.030667362 container died fec1006396f509075f84552bc3313fbcbb7b0563093929b75278bfa46d876d97 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-12T08:58:39.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:39 smithi114 ceph-mon[89482]: pgmap v304: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s; 26327/177510 objects degraded (14.831%); 8.0 KiB/s, 0 objects/s recovering 2024-04-12T08:58:39.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:39 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2286259740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:39.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 31 16 64592 64576 130.181 97.125 0.011221 0.00767721 2024-04-12T08:58:39.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:39 smithi090 ceph-mon[95814]: pgmap v304: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s; 26327/177510 objects degraded (14.831%); 8.0 KiB/s, 0 objects/s recovering 2024-04-12T08:58:39.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:39 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2286259740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:39.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:39 smithi090 ceph-mon[99575]: pgmap v304: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s; 26327/177510 objects degraded (14.831%); 8.0 KiB/s, 0 objects/s recovering 2024-04-12T08:58:39.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:39 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2286259740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:40.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 32 16 66105 66089 129.068 94.5625 0.00678751 0.00774285 2024-04-12T08:58:41.298 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:40 smithi090 podman[116666]: 2024-04-12 08:58:40.986097898 +0000 UTC m=+2.137002824 container remove fec1006396f509075f84552bc3313fbcbb7b0563093929b75278bfa46d876d97 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_CLEAN=True, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-12T08:58:41.580 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:41 smithi090 ceph-mon[95814]: pgmap v305: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 26327/177510 objects degraded (14.831%); 8.0 KiB/s, 0 objects/s recovering 2024-04-12T08:58:41.580 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:41 smithi090 ceph-mon[99575]: pgmap v305: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 26327/177510 objects degraded (14.831%); 8.0 KiB/s, 0 objects/s recovering 2024-04-12T08:58:41.581 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:41 smithi090 podman[116737]: 2024-04-12 08:58:41.20561389 +0000 UTC m=+0.034415252 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T08:58:41.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:41 smithi114 ceph-mon[89482]: pgmap v305: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 26327/177510 objects degraded (14.831%); 8.0 KiB/s, 0 objects/s recovering 2024-04-12T08:58:41.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:41 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2220738701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:41.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 33 16 67516 67500 127.829 88.1875 0.00833921 0.00781824 2024-04-12T08:58:41.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:41 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2220738701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:41.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:41 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2220738701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:42.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:42 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 27776/195540 objects degraded (14.205%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:42.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 34 16 68902 68886 126.617 86.625 0.0105332 0.0078933 2024-04-12T08:58:42.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:42 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 27776/195540 objects degraded (14.205%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:42.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:42 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 27776/195540 objects degraded (14.205%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:42.923 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:42 smithi090 podman[116737]: 2024-04-12 08:58:42.580949462 +0000 UTC m=+1.409750812 container create be4538ab3be1c3bb25d82b43a56ff08631a9e0587742a1312f60ba828f69cdf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240409, ceph=True) 2024-04-12T08:58:43.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:58:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:58:43] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2024-04-12T08:58:43.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:43 smithi090 ceph-mon[99575]: pgmap v306: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 27776/195540 objects degraded (14.205%); 7.2 KiB/s, 0 objects/s recovering 2024-04-12T08:58:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:43 smithi090 ceph-mon[95814]: pgmap v306: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 27776/195540 objects degraded (14.205%); 7.2 KiB/s, 0 objects/s recovering 2024-04-12T08:58:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:43 smithi114 ceph-mon[89482]: pgmap v306: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 27776/195540 objects degraded (14.205%); 7.2 KiB/s, 0 objects/s recovering 2024-04-12T08:58:43.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 35 16 70087 70071 125.115 74.0625 0.00796057 0.00798806 2024-04-12T08:58:44.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:44 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/347906847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:44.673 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:44 smithi090 podman[116737]: 2024-04-12 08:58:44.353210817 +0000 UTC m=+3.182012175 container init be4538ab3be1c3bb25d82b43a56ff08631a9e0587742a1312f60ba828f69cdf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2) 2024-04-12T08:58:44.673 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:44 smithi090 podman[116737]: 2024-04-12 08:58:44.363037782 +0000 UTC m=+3.191839132 container start be4538ab3be1c3bb25d82b43a56ff08631a9e0587742a1312f60ba828f69cdf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-12T08:58:44.673 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:44 smithi090 bash[116737]: be4538ab3be1c3bb25d82b43a56ff08631a9e0587742a1312f60ba828f69cdf8 2024-04-12T08:58:44.673 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:44 smithi090 systemd[1]: Started Ceph osd.1 for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T08:58:44.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:44 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/347906847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:44 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/347906847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:44.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 36 16 71699 71683 124.438 100.75 0.0071318 0.00803164 2024-04-12T08:58:45.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[116777]: 2024-04-12T08:58:44.922+0000 7f6f93a257c0 -1 Falling back to public interface 2024-04-12T08:58:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:45 smithi114 ceph-mon[89482]: pgmap v307: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 29726/201390 objects degraded (14.760%); 6.4 KiB/s, 0 objects/s recovering 2024-04-12T08:58:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:58:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:45.874 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 37 16 73118 73102 123.472 88.6875 0.0225589 0.00809379 2024-04-12T08:58:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[99575]: pgmap v307: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 29726/201390 objects degraded (14.760%); 6.4 KiB/s, 0 objects/s recovering 2024-04-12T08:58:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:58:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[95814]: pgmap v307: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 29726/201390 objects degraded (14.760%); 6.4 KiB/s, 0 objects/s recovering 2024-04-12T08:58:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:58:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:58:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:46 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3256884908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:46.875 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 38 16 74494 74478 122.484 86 0.0137543 0.00815963 2024-04-12T08:58:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:46 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3256884908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:46 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3256884908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:47.830 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:47 smithi090 ceph-mon[99575]: pgmap v308: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 30199/206805 objects degraded (14.603%); 5.4 KiB/s, 0 objects/s recovering 2024-04-12T08:58:47.831 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:47 smithi090 ceph-mon[95814]: pgmap v308: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 30199/206805 objects degraded (14.603%); 5.4 KiB/s, 0 objects/s recovering 2024-04-12T08:58:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:47 smithi114 ceph-mon[89482]: pgmap v308: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 30199/206805 objects degraded (14.603%); 5.4 KiB/s, 0 objects/s recovering 2024-04-12T08:58:47.875 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 39 16 75828 75812 121.481 83.375 0.0207606 0.00822706 2024-04-12T08:58:48.875 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:58:48.875741+0000 min lat: 0.00116602 max lat: 1.0518 avg lat: 0.0082938 2024-04-12T08:58:48.875 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:58:48.875 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 40 16 77138 77122 120.491 81.875 0.0211818 0.0082938 2024-04-12T08:58:48.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:48 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 32857/222471 objects degraded (14.769%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:48 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 32857/222471 objects degraded (14.769%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:48 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 32857/222471 objects degraded (14.769%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:49.875 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 41 15 78426 78411 119.517 80.5625 0.0123398 0.00836218 2024-04-12T08:58:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:49 smithi090 ceph-mon[99575]: pgmap v309: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s; 32857/222471 objects degraded (14.769%); 5.3 KiB/s, 0 objects/s recovering 2024-04-12T08:58:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:49 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3163284588' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:49 smithi090 ceph-mon[95814]: pgmap v309: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s; 32857/222471 objects degraded (14.769%); 5.3 KiB/s, 0 objects/s recovering 2024-04-12T08:58:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:49 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3163284588' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:49 smithi114 ceph-mon[89482]: pgmap v309: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s; 32857/222471 objects degraded (14.769%); 5.3 KiB/s, 0 objects/s recovering 2024-04-12T08:58:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:49 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3163284588' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:50.875 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 42 16 79708 79692 118.578 80.0625 0.0112633 0.00842848 2024-04-12T08:58:51.875 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 43 16 80883 80867 117.528 73.4375 0.0332614 0.00850298 2024-04-12T08:58:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:51 smithi090 ceph-mon[95814]: pgmap v310: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 32857/222471 objects degraded (14.769%) 2024-04-12T08:58:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:51 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:51 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:51 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1524270280' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:51 smithi090 ceph-mon[99575]: pgmap v310: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 32857/222471 objects degraded (14.769%) 2024-04-12T08:58:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:51 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:51 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:51 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1524270280' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:51 smithi114 ceph-mon[89482]: pgmap v310: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 32857/222471 objects degraded (14.769%) 2024-04-12T08:58:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:51 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:51 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:51 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1524270280' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:52.875 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 44 16 82126 82110 116.622 77.6875 0.008422 0.00856906 2024-04-12T08:58:53.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:58:53] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2024-04-12T08:58:53.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[116777]: 2024-04-12T08:58:53.150+0000 7f6f93a257c0 -1 osd.1 112 log_to_monitors true 2024-04-12T08:58:53.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[99575]: pgmap v311: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 33992/237012 objects degraded (14.342%) 2024-04-12T08:58:53.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.681 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[99575]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T08:58:53.681 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[95814]: pgmap v311: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 33992/237012 objects degraded (14.342%) 2024-04-12T08:58:53.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:53 smithi090 ceph-mon[95814]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T08:58:53.869 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:53 smithi114 ceph-mon[89482]: pgmap v311: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 33992/237012 objects degraded (14.342%) 2024-04-12T08:58:53.891 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:53 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.892 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:53 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.892 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:53 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.892 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:53 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:53.892 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:53 smithi114 ceph-mon[89482]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T08:58:53.892 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 45 16 83373 83357 115.762 77.9375 0.00813476 0.00863331 2024-04-12T08:58:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:54 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/510775363' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:54 smithi114 ceph-mon[89482]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T08:58:54.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:54 smithi114 ceph-mon[89482]: osdmap e115: 8 total, 7 up, 8 in 2024-04-12T08:58:54.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:54 smithi114 ceph-mon[89482]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:58:54.876 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 46 16 84355 84339 114.58 61.375 0.0168953 0.0087226 2024-04-12T08:58:54.922 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 08:58:54 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[116777]: 2024-04-12T08:58:54.708+0000 7f6f80e75700 -1 osd.1 112 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T08:58:54.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:54 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/510775363' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:54.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:54 smithi090 ceph-mon[95814]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T08:58:54.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:54 smithi090 ceph-mon[95814]: osdmap e115: 8 total, 7 up, 8 in 2024-04-12T08:58:54.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:54 smithi090 ceph-mon[95814]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:58:54.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:54 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/510775363' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:54.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:54 smithi090 ceph-mon[99575]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T08:58:54.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:54 smithi090 ceph-mon[99575]: osdmap e115: 8 total, 7 up, 8 in 2024-04-12T08:58:54.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:54 smithi090 ceph-mon[99575]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T08:58:55.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:55 smithi114 ceph-mon[89482]: pgmap v313: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 35590/241806 objects degraded (14.718%) 2024-04-12T08:58:55.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:55 smithi114 ceph-mon[89482]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' 2024-04-12T08:58:55.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:55 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 35590/241806 objects degraded (14.718%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:55.876 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 47 16 85300 85284 113.398 59.0625 0.0030746 0.00878845 2024-04-12T08:58:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:55 smithi090 ceph-mon[99575]: pgmap v313: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 35590/241806 objects degraded (14.718%) 2024-04-12T08:58:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:55 smithi090 ceph-mon[99575]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' 2024-04-12T08:58:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:55 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 35590/241806 objects degraded (14.718%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:55 smithi090 ceph-mon[95814]: pgmap v313: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 35590/241806 objects degraded (14.718%) 2024-04-12T08:58:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:55 smithi090 ceph-mon[95814]: from='osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530]' entity='osd.1' 2024-04-12T08:58:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:55 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 35590/241806 objects degraded (14.718%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T08:58:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:56 smithi114 ceph-mon[89482]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T08:58:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:56 smithi114 ceph-mon[89482]: osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530] boot 2024-04-12T08:58:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:56 smithi114 ceph-mon[89482]: osdmap e116: 8 total, 8 up, 8 in 2024-04-12T08:58:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:56 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:58:56.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:56 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1146554837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:56.876 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 48 16 85300 85284 111.036 0 - 0.00878845 2024-04-12T08:58:56.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:56 smithi090 ceph-mon[99575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T08:58:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:56 smithi090 ceph-mon[99575]: osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530] boot 2024-04-12T08:58:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:56 smithi090 ceph-mon[99575]: osdmap e116: 8 total, 8 up, 8 in 2024-04-12T08:58:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:56 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:58:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:56 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1146554837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:56 smithi090 ceph-mon[95814]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T08:58:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:56 smithi090 ceph-mon[95814]: osd.1 [v2:172.21.15.90:6810/1619969530,v1:172.21.15.90:6811/1619969530] boot 2024-04-12T08:58:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:56 smithi090 ceph-mon[95814]: osdmap e116: 8 total, 8 up, 8 in 2024-04-12T08:58:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:56 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T08:58:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:56 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1146554837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:58:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:57 smithi114 ceph-mon[89482]: pgmap v315: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s; 35958/246180 objects degraded (14.606%) 2024-04-12T08:58:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:57 smithi114 ceph-mon[89482]: osdmap e117: 8 total, 8 up, 8 in 2024-04-12T08:58:57.876 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 49 16 85300 85284 108.77 0 - 0.00878845 2024-04-12T08:58:57.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:57 smithi090 ceph-mon[99575]: pgmap v315: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s; 35958/246180 objects degraded (14.606%) 2024-04-12T08:58:57.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:57 smithi090 ceph-mon[99575]: osdmap e117: 8 total, 8 up, 8 in 2024-04-12T08:58:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:57 smithi090 ceph-mon[95814]: pgmap v315: 89 pgs: 11 active+undersized+degraded, 28 active+undersized, 50 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s; 35958/246180 objects degraded (14.606%) 2024-04-12T08:58:57.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:57 smithi090 ceph-mon[95814]: osdmap e117: 8 total, 8 up, 8 in 2024-04-12T08:58:58.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:58:58.226+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-12T08:58:58.876 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 50 16 85821 85805 107.246 10.8542 0.0254055 0.0093192 2024-04-12T08:58:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: osdmap e118: 8 total, 8 up, 8 in 2024-04-12T08:58:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: pgmap v318: 89 pgs: 5 remapped+peering, 14 peering, 4 active+undersized+degraded, 11 active+undersized, 55 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 10374/254982 objects degraded (4.069%) 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: pgmap v319: 89 pgs: 5 remapped+peering, 14 peering, 4 active+undersized+degraded, 11 active+undersized, 55 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 10374/254982 objects degraded (4.069%) 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:58:59.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:58 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-12T08:58:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: osdmap e118: 8 total, 8 up, 8 in 2024-04-12T08:58:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:58:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:58:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: pgmap v318: 89 pgs: 5 remapped+peering, 14 peering, 4 active+undersized+degraded, 11 active+undersized, 55 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 10374/254982 objects degraded (4.069%) 2024-04-12T08:58:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: pgmap v319: 89 pgs: 5 remapped+peering, 14 peering, 4 active+undersized+degraded, 11 active+undersized, 55 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 10374/254982 objects degraded (4.069%) 2024-04-12T08:58:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:58:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: osdmap e118: 8 total, 8 up, 8 in 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: pgmap v318: 89 pgs: 5 remapped+peering, 14 peering, 4 active+undersized+degraded, 11 active+undersized, 55 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 10374/254982 objects degraded (4.069%) 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: pgmap v319: 89 pgs: 5 remapped+peering, 14 peering, 4 active+undersized+degraded, 11 active+undersized, 55 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 10374/254982 objects degraded (4.069%) 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:58:59.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:58:59.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:58:59.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:58:59.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T08:58:59.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:58:59.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:58:59.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:58 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-12T08:58:59.876 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 51 16 86883 86867 106.444 66.375 0.0121081 0.00938902 2024-04-12T08:59:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:58:59 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3934718318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:00.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:58:59 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3934718318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:00.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:58:59 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3934718318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:00.876 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 52 16 87936 87920 105.663 65.8125 0.0107082 0.00945765 2024-04-12T08:59:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:00 smithi114 ceph-mon[89482]: pgmap v320: 89 pgs: 5 remapped+peering, 14 peering, 4 active+undersized+degraded, 2 active+undersized, 64 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s; 10374/254982 objects degraded (4.069%) 2024-04-12T08:59:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:00 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 10374/254982 objects degraded (4.069%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:00 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:59:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:00 smithi090 ceph-mon[99575]: pgmap v320: 89 pgs: 5 remapped+peering, 14 peering, 4 active+undersized+degraded, 2 active+undersized, 64 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s; 10374/254982 objects degraded (4.069%) 2024-04-12T08:59:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:00 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 10374/254982 objects degraded (4.069%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:00 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:59:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:00 smithi090 ceph-mon[95814]: pgmap v320: 89 pgs: 5 remapped+peering, 14 peering, 4 active+undersized+degraded, 2 active+undersized, 64 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s; 10374/254982 objects degraded (4.069%) 2024-04-12T08:59:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:00 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 10374/254982 objects degraded (4.069%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:00 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T08:59:01.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:01.876 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 53 16 88885 88869 104.788 59.3125 0.00579649 0.00953828 2024-04-12T08:59:02.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:01 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2072770303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:02.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:01 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2072770303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:02.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:01 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2072770303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:02.876 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 54 16 89884 89868 104.004 62.4375 0.0252414 0.00961006 2024-04-12T08:59:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:02 smithi114 ceph-mon[89482]: pgmap v321: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 3732/262092 objects degraded (1.424%) 2024-04-12T08:59:03.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:02 smithi090 ceph-mon[95814]: pgmap v321: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 3732/262092 objects degraded (1.424%) 2024-04-12T08:59:03.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:02 smithi090 ceph-mon[99575]: pgmap v321: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 3732/262092 objects degraded (1.424%) 2024-04-12T08:59:03.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:59:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:59:03] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.43.0" 2024-04-12T08:59:03.876 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 55 16 90878 90862 103.242 62.125 0.00754525 0.00968019 2024-04-12T08:59:04.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:04 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3380311852' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:04 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3380311852' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:04.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:04 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3380311852' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:04.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 56 15 91841 91826 102.474 60.25 0.0323786 0.00975383 2024-04-12T08:59:05.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:05 smithi090 ceph-mon[95814]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 13099/266124 objects degraded (4.922%); 905 KiB/s, 14 objects/s recovering 2024-04-12T08:59:05.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:05 smithi090 ceph-mon[99575]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 13099/266124 objects degraded (4.922%); 905 KiB/s, 14 objects/s recovering 2024-04-12T08:59:05.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:05 smithi114 ceph-mon[89482]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 13099/266124 objects degraded (4.922%); 905 KiB/s, 14 objects/s recovering 2024-04-12T08:59:05.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 57 16 92850 92834 101.782 63 0.0162625 0.00981962 2024-04-12T08:59:06.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:06 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 13099/266124 objects degraded (4.922%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:06.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:06 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 13099/266124 objects degraded (4.922%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:06.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:06 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 13099/266124 objects degraded (4.922%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:06.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 58 16 93780 93764 101.029 58.125 0.00833726 0.0098929 2024-04-12T08:59:07.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:07 smithi114 ceph-mon[89482]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 37 MiB/s wr, 591 op/s; 13099/269796 objects degraded (4.855%); 797 KiB/s, 12 objects/s recovering 2024-04-12T08:59:07.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:07 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/977191084' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:07.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:07 smithi090 ceph-mon[95814]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 37 MiB/s wr, 591 op/s; 13099/269796 objects degraded (4.855%); 797 KiB/s, 12 objects/s recovering 2024-04-12T08:59:07.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:07 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/977191084' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:07 smithi090 ceph-mon[99575]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 37 MiB/s wr, 591 op/s; 13099/269796 objects degraded (4.855%); 797 KiB/s, 12 objects/s recovering 2024-04-12T08:59:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:07 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/977191084' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:07.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 59 16 94769 94753 100.364 61.8125 0.0121199 0.00995857 2024-04-12T08:59:08.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:59:08.877556+0000 min lat: 0.00116602 max lat: 2.6053 avg lat: 0.0100238 2024-04-12T08:59:08.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:59:08.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 60 16 95752 95736 99.7154 61.4375 0.00987457 0.0100238 2024-04-12T08:59:09.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:09 smithi114 ceph-mon[89482]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 12951/280803 objects degraded (4.612%); 1.6 MiB/s, 25 objects/s recovering 2024-04-12T08:59:09.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:09 smithi090 ceph-mon[95814]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 12951/280803 objects degraded (4.612%); 1.6 MiB/s, 25 objects/s recovering 2024-04-12T08:59:09.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:09 smithi090 ceph-mon[99575]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 12951/280803 objects degraded (4.612%); 1.6 MiB/s, 25 objects/s recovering 2024-04-12T08:59:09.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 61 16 96684 96668 99.0355 58.25 0.00664628 0.0100926 2024-04-12T08:59:10.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:10 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3481772654' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:10.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:10 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3481772654' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:10.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:10 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3481772654' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:10.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 62 16 97669 97653 98.431 61.5625 0.0116291 0.0101537 2024-04-12T08:59:11.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:11 smithi114 ceph-mon[89482]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 12951/280803 objects degraded (4.612%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:59:11.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:11 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 12951/280803 objects degraded (4.612%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:11.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:11 smithi090 ceph-mon[99575]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 12951/280803 objects degraded (4.612%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:59:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:11 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 12951/280803 objects degraded (4.612%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:11.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:11 smithi090 ceph-mon[95814]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 12951/280803 objects degraded (4.612%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:59:11.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:11 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 12951/280803 objects degraded (4.612%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:11.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 63 16 98580 98564 97.7723 56.9375 0.0144617 0.0102213 2024-04-12T08:59:12.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:12 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/324753125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:12.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:12 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/324753125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:12.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:12 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/324753125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:12.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 64 16 99532 99516 97.1742 59.5 0.0130337 0.0102847 2024-04-12T08:59:13.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:13 smithi114 ceph-mon[89482]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 12951/290688 objects degraded (4.455%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:59:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:59:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:59:13.243+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:59:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:59:13] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.43.0" 2024-04-12T08:59:13.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:13 smithi090 ceph-mon[99575]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 12951/290688 objects degraded (4.455%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:59:13.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:13 smithi090 ceph-mon[95814]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 12951/290688 objects degraded (4.455%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T08:59:13.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 65 16 100528 100512 96.6368 62.25 0.00972322 0.0103427 2024-04-12T08:59:14.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:14 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:14.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:14 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:14.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:14 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:14.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:14.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:14 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:14.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:14 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:14.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:14.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:14 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:14.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:14 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:14.877 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 66 16 101412 101396 96.0097 55.25 0.00713375 0.0104101 2024-04-12T08:59:15.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:15 smithi114 ceph-mon[89482]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 12811/295272 objects degraded (4.339%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T08:59:15.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:15 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/534474842' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:15.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:15 smithi090 ceph-mon[99575]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 12811/295272 objects degraded (4.339%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T08:59:15.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:15 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/534474842' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:15.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:15 smithi090 ceph-mon[95814]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 12811/295272 objects degraded (4.339%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T08:59:15.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:15 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/534474842' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:15.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 67 16 102305 102289 95.4097 55.8125 0.0246862 0.0104752 2024-04-12T08:59:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:16 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 12811/295272 objects degraded (4.339%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:16 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:16.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:16 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 12811/295272 objects degraded (4.339%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:16.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:16 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:16.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:16 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 12811/295272 objects degraded (4.339%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:16.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:16.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 68 16 103180 103164 94.8107 54.6875 0.0348025 0.0105413 2024-04-12T08:59:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:17 smithi114 ceph-mon[89482]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s; 12811/298737 objects degraded (4.288%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T08:59:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:17 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/606981761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:17.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:17 smithi090 ceph-mon[99575]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s; 12811/298737 objects degraded (4.288%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T08:59:17.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:17 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/606981761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:17.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:17 smithi090 ceph-mon[95814]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s; 12811/298737 objects degraded (4.288%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T08:59:17.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:17 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/606981761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:17.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 69 16 104079 104063 94.2509 56.1875 0.00532569 0.010604 2024-04-12T08:59:18.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 70 16 105002 104986 93.7285 57.6875 0.0136971 0.0106636 2024-04-12T08:59:19.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:19 smithi114 ceph-mon[89482]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 12670/309087 objects degraded (4.099%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T08:59:19.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:19 smithi090 ceph-mon[99575]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 12670/309087 objects degraded (4.099%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T08:59:19.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:19 smithi090 ceph-mon[95814]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 12670/309087 objects degraded (4.099%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T08:59:19.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 71 16 105880 105864 93.1812 54.875 0.013388 0.0107265 2024-04-12T08:59:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:20 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1411993759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:20.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:20 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1411993759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:20.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:20 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1411993759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:20.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 72 16 106741 106725 92.6344 53.8125 0.00573141 0.0107901 2024-04-12T08:59:21.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:21 smithi114 ceph-mon[89482]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s; 12670/309087 objects degraded (4.099%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T08:59:21.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:21 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 12670/309087 objects degraded (4.099%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:21.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:21 smithi090 ceph-mon[99575]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s; 12670/309087 objects degraded (4.099%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T08:59:21.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:21 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 12670/309087 objects degraded (4.099%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:21.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:21 smithi090 ceph-mon[95814]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s; 12670/309087 objects degraded (4.099%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T08:59:21.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:21 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 12670/309087 objects degraded (4.099%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:21.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 73 16 107666 107650 92.1573 57.8125 0.00492337 0.0108457 2024-04-12T08:59:22.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:22 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2272075427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:22.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:22 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2272075427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:22.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:22 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2272075427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:22.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 74 16 108530 108514 91.6416 54 0.0088871 0.0109059 2024-04-12T08:59:23.575 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:23 smithi090 ceph-mon[95814]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 12670/318417 objects degraded (3.979%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T08:59:23.575 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:59:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:59:23] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2024-04-12T08:59:23.575 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:23 smithi090 ceph-mon[99575]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 12670/318417 objects degraded (3.979%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T08:59:23.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:23 smithi114 ceph-mon[89482]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 12670/318417 objects degraded (3.979%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T08:59:23.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 75 16 109400 109384 91.1447 54.375 0.0158769 0.0109661 2024-04-12T08:59:24.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 76 16 110140 110124 90.5539 46.25 0.0155776 0.011037 2024-04-12T08:59:25.551 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:25 smithi090 ceph-mon[99575]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 12512/322518 objects degraded (3.879%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T08:59:25.551 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4001819172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:25.551 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:25 smithi090 ceph-mon[95814]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 12512/322518 objects degraded (3.879%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T08:59:25.551 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4001819172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:25.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:25 smithi114 ceph-mon[89482]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 12512/322518 objects degraded (3.879%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T08:59:25.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4001819172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:25.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 77 16 110976 110960 90.0564 52.25 0.0237175 0.0110992 2024-04-12T08:59:26.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:26 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 12512/322518 objects degraded (3.879%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:26.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:26 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 12512/322518 objects degraded (3.879%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:26 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 12512/322518 objects degraded (3.879%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:26.878 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 78 16 111856 111840 89.6069 55 0.0134306 0.0111539 2024-04-12T08:59:27.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:27 smithi114 ceph-mon[89482]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 12512/325572 objects degraded (3.843%); 1.6 MiB/s, 24 objects/s recovering 2024-04-12T08:59:27.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:27 smithi090 ceph-mon[95814]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 12512/325572 objects degraded (3.843%); 1.6 MiB/s, 24 objects/s recovering 2024-04-12T08:59:27.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:27 smithi090 ceph-mon[99575]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 12512/325572 objects degraded (3.843%); 1.6 MiB/s, 24 objects/s recovering 2024-04-12T08:59:27.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 79 16 112681 112665 89.1253 51.5625 0.00996095 0.0112149 2024-04-12T08:59:28.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:28 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/800362502' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:28.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:28 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:28.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:59:28 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:59:28.261+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:28.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:28 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/800362502' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:28.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:28 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:28.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:28 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/800362502' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:28.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:28 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:28.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:2024-04-12T08:59:28.879306+0000 min lat: 0.00116602 max lat: 2.6053 avg lat: 0.0112713 2024-04-12T08:59:28.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T08:59:28.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 80 16 113537 113521 88.6799 53.5 0.010935 0.0112713 2024-04-12T08:59:29.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:29 smithi114 ceph-mon[89482]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 12336/334971 objects degraded (3.683%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T08:59:29.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:29 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:29.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:29 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:29.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:29 smithi090 ceph-mon[95814]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 12336/334971 objects degraded (3.683%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T08:59:29.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:29 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:29.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:29 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:29.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:29 smithi090 ceph-mon[99575]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 12336/334971 objects degraded (3.683%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T08:59:29.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:29 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:29.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:29 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:29.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 81 16 114416 114400 88.2633 54.9375 0.0139509 0.0113244 2024-04-12T08:59:30.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:30 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3351527933' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:30.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:30 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3351527933' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:30.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:30 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3351527933' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:30.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 82 16 115251 115235 87.8233 52.1875 0.0188505 0.0113813 2024-04-12T08:59:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:31 smithi114 ceph-mon[89482]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 12336/334971 objects degraded (3.683%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T08:59:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:31 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 12336/334971 objects degraded (3.683%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:31 smithi090 ceph-mon[95814]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 12336/334971 objects degraded (3.683%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T08:59:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:31 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 12336/334971 objects degraded (3.683%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:31 smithi090 ceph-mon[99575]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 12336/334971 objects degraded (3.683%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T08:59:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:31 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 12336/334971 objects degraded (3.683%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:31.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 83 15 116135 116120 87.4315 55.3125 0.0176201 0.0114321 2024-04-12T08:59:32.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 84 16 116945 116929 86.9925 50.5625 0.00824387 0.0114896 2024-04-12T08:59:33.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:33 smithi114 ceph-mon[89482]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s; 12336/343761 objects degraded (3.589%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T08:59:33.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:33 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2701105467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:59:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:59:33] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2024-04-12T08:59:33.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:33 smithi090 ceph-mon[95814]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s; 12336/343761 objects degraded (3.589%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T08:59:33.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:33 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2701105467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:33.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:33 smithi090 ceph-mon[99575]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s; 12336/343761 objects degraded (3.589%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T08:59:33.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:33 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2701105467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:33.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 85 15 117802 117787 86.5999 53.625 0.0195502 0.0115408 2024-04-12T08:59:34.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 86 16 118670 118654 86.223 54.1875 0.0333633 0.0115925 2024-04-12T08:59:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:35 smithi114 ceph-mon[89482]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 12159/347883 objects degraded (3.495%); 2.7 MiB/s, 42 objects/s recovering 2024-04-12T08:59:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:35 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1698029452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:35.648 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:35 smithi090 ceph-mon[99575]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 12159/347883 objects degraded (3.495%); 2.7 MiB/s, 42 objects/s recovering 2024-04-12T08:59:35.649 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:35 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1698029452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:35.649 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:35 smithi090 ceph-mon[95814]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 12159/347883 objects degraded (3.495%); 2.7 MiB/s, 42 objects/s recovering 2024-04-12T08:59:35.649 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:35 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1698029452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:35.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 87 16 119455 119439 85.7958 49.0625 0.0130026 0.0116502 2024-04-12T08:59:36.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:36 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 12159/347883 objects degraded (3.495%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:36.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:36 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 12159/347883 objects degraded (3.495%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:36.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:36 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 12159/347883 objects degraded (3.495%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:36.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 88 16 120288 120272 85.4124 52.0625 0.0190944 0.0117023 2024-04-12T08:59:37.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:37 smithi114 ceph-mon[89482]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 12159/351027 objects degraded (3.464%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T08:59:37.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:37 smithi090 ceph-mon[99575]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 12159/351027 objects degraded (3.464%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T08:59:37.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:37 smithi090 ceph-mon[95814]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 12159/351027 objects degraded (3.464%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T08:59:37.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 89 16 121077 121061 85.0068 49.3125 0.0205218 0.0117584 2024-04-12T08:59:38.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:38 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2265956704' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:38.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:38 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2265956704' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:38.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:38 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2265956704' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:38.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout: 90 13 121923 121910 84.6518 53.0625 0.0081461 0.011807 2024-04-12T08:59:39.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:39 smithi090 ceph-mon[99575]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 11995/360426 objects degraded (3.328%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T08:59:39.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:39 smithi090 ceph-mon[95814]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 11995/360426 objects degraded (3.328%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T08:59:39.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:39 smithi114 ceph-mon[89482]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 11995/360426 objects degraded (3.328%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T08:59:39.879 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Total time run: 90.0192 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Total writes made: 121923 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Write size: 65536 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Object size: 65536 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Bandwidth (MB/sec): 84.6507 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev Bandwidth: 46.1176 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max bandwidth (MB/sec): 247.625 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min bandwidth (MB/sec): 0 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Average IOPS: 1354 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev IOPS: 737.894 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max IOPS: 3962 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min IOPS: 0 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Average Latency(s): 0.0118082 2024-04-12T08:59:39.880 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Stddev Latency(s): 0.0328352 2024-04-12T08:59:39.881 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Max latency(s): 2.6053 2024-04-12T08:59:39.881 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Min latency(s): 0.00116602 2024-04-12T08:59:39.888 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Cleaning up (deleting benchmark objects) 2024-04-12T08:59:41.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:41 smithi090 ceph-mon[99575]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 11995/360426 objects degraded (3.328%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T08:59:41.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:41 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 11995/360426 objects degraded (3.328%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:41.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:41 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2999476771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:41 smithi090 ceph-mon[95814]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 11995/360426 objects degraded (3.328%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T08:59:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:41 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 11995/360426 objects degraded (3.328%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:41 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2999476771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:41.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:41 smithi114 ceph-mon[89482]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 11995/360426 objects degraded (3.328%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T08:59:41.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:41 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 11995/360426 objects degraded (3.328%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:41.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:41 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2999476771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:43.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:43 smithi090 ceph-mon[95814]: pgmap v341: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 894 op/s; 11995/362451 objects degraded (3.309%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T08:59:43.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:43 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2667818454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:43.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:43 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:43.675 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:59:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T08:59:43.279+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:43.675 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:59:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:59:43] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2024-04-12T08:59:43.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:43 smithi090 ceph-mon[99575]: pgmap v341: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 894 op/s; 11995/362451 objects degraded (3.309%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T08:59:43.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:43 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2667818454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:43.675 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:43 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:43 smithi114 ceph-mon[89482]: pgmap v341: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 894 op/s; 11995/362451 objects degraded (3.309%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T08:59:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:43 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2667818454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:43 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:44.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:44 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:44.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:44 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:44.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:44 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:44.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:44 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:44 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T08:59:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:44 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T08:59:45.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:45 smithi090 ceph-mon[99575]: pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 738 op/s; 11536/361332 objects degraded (3.193%); 4.2 MiB/s, 66 objects/s recovering 2024-04-12T08:59:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:45 smithi090 ceph-mon[95814]: pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 738 op/s; 11536/361332 objects degraded (3.193%); 4.2 MiB/s, 66 objects/s recovering 2024-04-12T08:59:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:45 smithi114 ceph-mon[89482]: pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 738 op/s; 11536/361332 objects degraded (3.193%); 4.2 MiB/s, 66 objects/s recovering 2024-04-12T08:59:46.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:46 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 11536/361332 objects degraded (3.193%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:46.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:46 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:46.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:46 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2586255133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:46 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 11536/361332 objects degraded (3.193%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:46 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:46 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2586255133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:46 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 11536/361332 objects degraded (3.193%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:46 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T08:59:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:46 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2586255133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:47 smithi114 ceph-mon[89482]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 713 op/s; 11536/358122 objects degraded (3.221%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T08:59:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:47 smithi090 ceph-mon[99575]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 713 op/s; 11536/358122 objects degraded (3.221%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T08:59:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:47 smithi090 ceph-mon[95814]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 713 op/s; 11536/358122 objects degraded (3.221%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T08:59:48.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:48 smithi114 ceph-mon[89482]: osdmap e119: 8 total, 8 up, 8 in 2024-04-12T08:59:48.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:48 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1187484896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:48.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:48 smithi090 ceph-mon[99575]: osdmap e119: 8 total, 8 up, 8 in 2024-04-12T08:59:48.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:48 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1187484896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:48 smithi090 ceph-mon[95814]: osdmap e119: 8 total, 8 up, 8 in 2024-04-12T08:59:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:48 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1187484896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:49.768 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:49 smithi114 ceph-mon[89482]: pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 11 MiB/s wr, 729 op/s; 11536/349365 objects degraded (3.302%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T08:59:49.768 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:49 smithi114 ceph-mon[89482]: osdmap e120: 8 total, 8 up, 8 in 2024-04-12T08:59:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:49 smithi090 ceph-mon[99575]: pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 11 MiB/s wr, 729 op/s; 11536/349365 objects degraded (3.302%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T08:59:49.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:49 smithi090 ceph-mon[99575]: osdmap e120: 8 total, 8 up, 8 in 2024-04-12T08:59:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:49 smithi090 ceph-mon[95814]: pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 11 MiB/s wr, 729 op/s; 11536/349365 objects degraded (3.302%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T08:59:49.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:49 smithi090 ceph-mon[95814]: osdmap e120: 8 total, 8 up, 8 in 2024-04-12T08:59:51.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:51 smithi114 ceph-mon[89482]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 2.7 MiB/s wr, 668 op/s; 11360/348447 objects degraded (3.260%); 5.0 MiB/s, 79 objects/s recovering 2024-04-12T08:59:51.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:51 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 11360/348447 objects degraded (3.260%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:51.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:51 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2955834034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:51 smithi090 ceph-mon[99575]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 2.7 MiB/s wr, 668 op/s; 11360/348447 objects degraded (3.260%); 5.0 MiB/s, 79 objects/s recovering 2024-04-12T08:59:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:51 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 11360/348447 objects degraded (3.260%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:51 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2955834034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:51 smithi090 ceph-mon[95814]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 2.7 MiB/s wr, 668 op/s; 11360/348447 objects degraded (3.260%); 5.0 MiB/s, 79 objects/s recovering 2024-04-12T08:59:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:51 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 11360/348447 objects degraded (3.260%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:51 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2955834034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 08:59:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:08:59:53] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2024-04-12T08:59:53.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:53 smithi090 ceph-mon[99575]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 11068/340482 objects degraded (3.251%); 3.7 MiB/s, 58 objects/s recovering 2024-04-12T08:59:53.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:53 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1705371257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:53.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:53 smithi090 ceph-mon[95814]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 11068/340482 objects degraded (3.251%); 3.7 MiB/s, 58 objects/s recovering 2024-04-12T08:59:53.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:53 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1705371257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:53 smithi114 ceph-mon[89482]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 11068/340482 objects degraded (3.251%); 3.7 MiB/s, 58 objects/s recovering 2024-04-12T08:59:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:53 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1705371257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:55.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:55 smithi114 ceph-mon[89482]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 11068/337659 objects degraded (3.278%); 3.7 MiB/s, 58 objects/s recovering 2024-04-12T08:59:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:55 smithi090 ceph-mon[99575]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 11068/337659 objects degraded (3.278%); 3.7 MiB/s, 58 objects/s recovering 2024-04-12T08:59:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:55 smithi090 ceph-mon[95814]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 11068/337659 objects degraded (3.278%); 3.7 MiB/s, 58 objects/s recovering 2024-04-12T08:59:56.627 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:56 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 11068/337659 objects degraded (3.278%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:56.627 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:56 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1640930915' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:56.628 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:56 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 11068/337659 objects degraded (3.278%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:56.628 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:56 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1640930915' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:56 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 11068/337659 objects degraded (3.278%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T08:59:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:56 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1640930915' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:57 smithi114 ceph-mon[89482]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 11068/333696 objects degraded (3.317%); 3.4 MiB/s, 53 objects/s recovering 2024-04-12T08:59:57.868 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:57 smithi090 ceph-mon[99575]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 11068/333696 objects degraded (3.317%); 3.4 MiB/s, 53 objects/s recovering 2024-04-12T08:59:57.868 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:57 smithi090 ceph-mon[95814]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 11068/333696 objects degraded (3.317%); 3.4 MiB/s, 53 objects/s recovering 2024-04-12T08:59:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:59:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:58 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1812742949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:58.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:59:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:58 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1812742949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:58.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T08:59:58.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:58 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1812742949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T08:59:59.738 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 08:59:59 smithi114 ceph-mon[89482]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 10839/325428 objects degraded (3.331%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T08:59:59.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 08:59:59 smithi090 ceph-mon[95814]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 10839/325428 objects degraded (3.331%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T08:59:59.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 08:59:59 smithi090 ceph-mon[99575]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 10839/325428 objects degraded (3.331%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T09:00:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 10839/325428 objects degraded (3.331%), 6 pgs degraded, 6 pgs undersized 2024-04-12T09:00:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T09:00:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: daemon grafana.a on smithi114 is in error state 2024-04-12T09:00:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: [WRN] PG_DEGRADED: Degraded data redundancy: 10839/325428 objects degraded (3.331%), 6 pgs degraded, 6 pgs undersized 2024-04-12T09:00:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: pg 8.0 is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2024-04-12T09:00:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: pg 8.1 is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,2] 2024-04-12T09:00:00.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: pg 8.3 is stuck undersized for 60s, current state active+recovering+undersized+degraded+remapped, last acting [5,6] 2024-04-12T09:00:00.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: pg 8.a is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2024-04-12T09:00:00.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: pg 8.b is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,4] 2024-04-12T09:00:00.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: pg 8.f is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2] 2024-04-12T09:00:00.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 10839/325428 objects degraded (3.331%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:00.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:00.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 10839/325428 objects degraded (3.331%), 6 pgs degraded, 6 pgs undersized 2024-04-12T09:00:00.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T09:00:00.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: daemon grafana.a on smithi114 is in error state 2024-04-12T09:00:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: [WRN] PG_DEGRADED: Degraded data redundancy: 10839/325428 objects degraded (3.331%), 6 pgs degraded, 6 pgs undersized 2024-04-12T09:00:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: pg 8.0 is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2024-04-12T09:00:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: pg 8.1 is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,2] 2024-04-12T09:00:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: pg 8.3 is stuck undersized for 60s, current state active+recovering+undersized+degraded+remapped, last acting [5,6] 2024-04-12T09:00:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: pg 8.a is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2024-04-12T09:00:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: pg 8.b is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,4] 2024-04-12T09:00:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: pg 8.f is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2] 2024-04-12T09:00:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 10839/325428 objects degraded (3.331%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:00.675 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 10839/325428 objects degraded (3.331%), 6 pgs degraded, 6 pgs undersized 2024-04-12T09:00:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T09:00:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: daemon grafana.a on smithi114 is in error state 2024-04-12T09:00:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: [WRN] PG_DEGRADED: Degraded data redundancy: 10839/325428 objects degraded (3.331%), 6 pgs degraded, 6 pgs undersized 2024-04-12T09:00:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: pg 8.0 is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2024-04-12T09:00:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: pg 8.1 is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,2] 2024-04-12T09:00:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: pg 8.3 is stuck undersized for 60s, current state active+recovering+undersized+degraded+remapped, last acting [5,6] 2024-04-12T09:00:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: pg 8.a is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2024-04-12T09:00:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: pg 8.b is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,4] 2024-04-12T09:00:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: pg 8.f is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2] 2024-04-12T09:00:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 10839/325428 objects degraded (3.331%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:00.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:01 smithi114 ceph-mon[89482]: pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 10839/324711 objects degraded (3.338%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T09:00:01.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:01 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3963049715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:01.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:01 smithi090 ceph-mon[99575]: pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 10839/324711 objects degraded (3.338%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T09:00:01.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:01 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3963049715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:01.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:01 smithi090 ceph-mon[95814]: pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 10839/324711 objects degraded (3.338%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T09:00:01.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:01 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3963049715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:03.372 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:00:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:00:03] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.43.0" 2024-04-12T09:00:03.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:03 smithi090 ceph-mon[99575]: pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 10613/317067 objects degraded (3.347%); 3.9 MiB/s, 62 objects/s recovering 2024-04-12T09:00:03.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:03 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:03.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:03 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:03.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:03 smithi090 ceph-mon[95814]: pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 10613/317067 objects degraded (3.347%); 3.9 MiB/s, 62 objects/s recovering 2024-04-12T09:00:03.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:03 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:03.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:03 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:03 smithi114 ceph-mon[89482]: pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 10613/317067 objects degraded (3.347%); 3.9 MiB/s, 62 objects/s recovering 2024-04-12T09:00:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:03 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:03.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:03 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:04.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:04 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1172612212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:04.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:04 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1172612212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:04 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1172612212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:05 smithi114 ceph-mon[89482]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 10613/314424 objects degraded (3.375%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T09:00:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:05 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:05 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:05 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 10613/314424 objects degraded (3.375%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:05 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:00:05.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:05 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:00:05.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:00:05.735+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:05.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[99575]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 10613/314424 objects degraded (3.375%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T09:00:05.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:05.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:05.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 10613/314424 objects degraded (3.375%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:05.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:00:05.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:00:05.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[95814]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 10613/314424 objects degraded (3.375%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T09:00:05.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:05.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:05.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 10613/314424 objects degraded (3.375%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:05.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:00:05.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:05 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:00:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:06 smithi114 ceph-mon[89482]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 10613/313725 objects degraded (3.383%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T09:00:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:06 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:06 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:00:06.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:06 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:00:06.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:06 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:00:06.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:06 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:00:06.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:06 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:06.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:06 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/745389916' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[99575]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 10613/313725 objects degraded (3.383%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T09:00:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:00:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:00:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:00:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:00:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/745389916' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:06.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[95814]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 10613/313725 objects degraded (3.383%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T09:00:06.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:00:06.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:00:06.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:00:06.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:00:06.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:00:06.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:06.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:06 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/745389916' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:07.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:07 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:07.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:07 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:07.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:07 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:07.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:07 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:07.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:07 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:07.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:07 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:08 smithi114 ceph-mon[89482]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 10387/305277 objects degraded (3.402%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T09:00:08.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:08 smithi090 ceph-mon[99575]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 10387/305277 objects degraded (3.402%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T09:00:08.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:08 smithi090 ceph-mon[95814]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 10387/305277 objects degraded (3.402%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T09:00:09.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:09 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1174314938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:09.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:09 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1174314938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:09.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:09 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1174314938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:10 smithi114 ceph-mon[89482]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 10387/302346 objects degraded (3.435%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T09:00:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:10 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 10387/302346 objects degraded (3.435%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:10 smithi090 ceph-mon[99575]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 10387/302346 objects degraded (3.435%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T09:00:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:10 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 10387/302346 objects degraded (3.435%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:10 smithi090 ceph-mon[95814]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 10387/302346 objects degraded (3.435%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T09:00:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:10 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 10387/302346 objects degraded (3.435%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:11 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/210322478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:11.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:11 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/210322478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:11.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:11 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/210322478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:12 smithi114 ceph-mon[89482]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 10387/299247 objects degraded (3.471%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T09:00:12.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:12 smithi090 ceph-mon[99575]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 10387/299247 objects degraded (3.471%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T09:00:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:12 smithi090 ceph-mon[95814]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 10387/299247 objects degraded (3.471%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T09:00:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:00:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:00:13] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.43.0" 2024-04-12T09:00:14.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:14 smithi114 ceph-mon[89482]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 10151/290958 objects degraded (3.489%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T09:00:14.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:14 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2398410787' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:14 smithi090 ceph-mon[95814]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 10151/290958 objects degraded (3.489%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T09:00:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:14 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2398410787' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:14.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:14 smithi090 ceph-mon[99575]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 10151/290958 objects degraded (3.489%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T09:00:14.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:14 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2398410787' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:15 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 10151/290958 objects degraded (3.489%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:15 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:15 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 10151/290958 objects degraded (3.489%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:16.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:15 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 10151/290958 objects degraded (3.489%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:16.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:15 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:16 smithi090 ceph-mon[95814]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 10151/290190 objects degraded (3.498%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T09:00:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:16 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2593378486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:16 smithi090 ceph-mon[99575]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 10151/290190 objects degraded (3.498%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T09:00:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:16 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2593378486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:16 smithi114 ceph-mon[89482]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 10151/290190 objects degraded (3.498%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T09:00:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:16 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2593378486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:18.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:18 smithi090 ceph-mon[95814]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 9874/281121 objects degraded (3.512%); 3.8 MiB/s, 61 objects/s recovering 2024-04-12T09:00:18.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:18 smithi090 ceph-mon[99575]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 9874/281121 objects degraded (3.512%); 3.8 MiB/s, 61 objects/s recovering 2024-04-12T09:00:19.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:18 smithi114 ceph-mon[89482]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 9874/281121 objects degraded (3.512%); 3.8 MiB/s, 61 objects/s recovering 2024-04-12T09:00:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:19 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1973064246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:19.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:19 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1973064246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:20.017 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:19 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1973064246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:20.804 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:20 smithi114 ceph-mon[89482]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 9874/278148 objects degraded (3.550%); 2.7 MiB/s, 42 objects/s recovering 2024-04-12T09:00:20.804 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:20 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 9874/278148 objects degraded (3.550%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:20 smithi090 ceph-mon[95814]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 9874/278148 objects degraded (3.550%); 2.7 MiB/s, 42 objects/s recovering 2024-04-12T09:00:20.947 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:20 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 9874/278148 objects degraded (3.550%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:20.948 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:00:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:00:20.752+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:20.948 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:20 smithi090 ceph-mon[99575]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 9874/278148 objects degraded (3.550%); 2.7 MiB/s, 42 objects/s recovering 2024-04-12T09:00:20.948 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:20 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 9874/278148 objects degraded (3.550%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:21.899 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:21 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:21.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:21 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:21.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:21 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:22.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:22 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:22.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:22 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:22.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:22 smithi090 ceph-mon[95814]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 9874/275262 objects degraded (3.587%); 2.7 MiB/s, 42 objects/s recovering 2024-04-12T09:00:22.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:22 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/556291133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:22.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:22 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:22.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:22 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:22.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:22 smithi090 ceph-mon[99575]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 9874/275262 objects degraded (3.587%); 2.7 MiB/s, 42 objects/s recovering 2024-04-12T09:00:22.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:22 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/556291133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:22 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:22 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:22 smithi114 ceph-mon[89482]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 9874/275262 objects degraded (3.587%); 2.7 MiB/s, 42 objects/s recovering 2024-04-12T09:00:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:22 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/556291133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:23.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:00:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:00:23] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2024-04-12T09:00:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:23 smithi114 ceph-mon[89482]: osdmap e121: 8 total, 8 up, 8 in 2024-04-12T09:00:24.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:23 smithi090 ceph-mon[99575]: osdmap e121: 8 total, 8 up, 8 in 2024-04-12T09:00:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:23 smithi090 ceph-mon[95814]: osdmap e121: 8 total, 8 up, 8 in 2024-04-12T09:00:25.105 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:24 smithi090 ceph-mon[95814]: pgmap v365: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 9563/267438 objects degraded (3.576%); 3.7 MiB/s, 58 objects/s recovering 2024-04-12T09:00:25.105 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:24 smithi090 ceph-mon[95814]: osdmap e122: 8 total, 8 up, 8 in 2024-04-12T09:00:25.105 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:24 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2064111483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:25.105 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:24 smithi090 ceph-mon[99575]: pgmap v365: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 9563/267438 objects degraded (3.576%); 3.7 MiB/s, 58 objects/s recovering 2024-04-12T09:00:25.105 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:24 smithi090 ceph-mon[99575]: osdmap e122: 8 total, 8 up, 8 in 2024-04-12T09:00:25.106 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:24 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2064111483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:25.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:24 smithi114 ceph-mon[89482]: pgmap v365: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 9563/267438 objects degraded (3.576%); 3.7 MiB/s, 58 objects/s recovering 2024-04-12T09:00:25.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:24 smithi114 ceph-mon[89482]: osdmap e122: 8 total, 8 up, 8 in 2024-04-12T09:00:25.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:24 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2064111483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:25 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 9563/267438 objects degraded (3.576%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:26.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:25 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 9563/267438 objects degraded (3.576%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:26.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:25 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 9563/267438 objects degraded (3.576%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:26 smithi090 ceph-mon[99575]: pgmap v367: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 9550/266487 objects degraded (3.584%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T09:00:26.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:26 smithi090 ceph-mon[95814]: pgmap v367: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 9550/266487 objects degraded (3.584%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T09:00:27.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:26 smithi114 ceph-mon[89482]: pgmap v367: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 9550/266487 objects degraded (3.584%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T09:00:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:27 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/269560961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:28.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:27 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/269560961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:27 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/269560961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:29.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:28 smithi114 ceph-mon[89482]: pgmap v368: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 9301/260013 objects degraded (3.577%); 4.5 MiB/s, 71 objects/s recovering 2024-04-12T09:00:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:28 smithi090 ceph-mon[95814]: pgmap v368: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 9301/260013 objects degraded (3.577%); 4.5 MiB/s, 71 objects/s recovering 2024-04-12T09:00:29.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:28 smithi090 ceph-mon[99575]: pgmap v368: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 9301/260013 objects degraded (3.577%); 4.5 MiB/s, 71 objects/s recovering 2024-04-12T09:00:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:29 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2118449862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:29 smithi114 ceph-mon[89482]: pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 9301/256851 objects degraded (3.621%); 4.5 MiB/s, 71 objects/s recovering 2024-04-12T09:00:30.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:29 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2118449862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:30.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:29 smithi090 ceph-mon[99575]: pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 9301/256851 objects degraded (3.621%); 4.5 MiB/s, 71 objects/s recovering 2024-04-12T09:00:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:29 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2118449862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:29 smithi090 ceph-mon[95814]: pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 9301/256851 objects degraded (3.621%); 4.5 MiB/s, 71 objects/s recovering 2024-04-12T09:00:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:30 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 9301/256851 objects degraded (3.621%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:30 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:31.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:30 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 9301/256851 objects degraded (3.621%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:31.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:30 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:31.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:30 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 9301/256851 objects degraded (3.621%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:31.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:30 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:32.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:31 smithi114 ceph-mon[89482]: pgmap v370: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 9301/253896 objects degraded (3.663%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T09:00:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:31 smithi090 ceph-mon[99575]: pgmap v370: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 9301/253896 objects degraded (3.663%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T09:00:32.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:31 smithi090 ceph-mon[95814]: pgmap v370: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 9301/253896 objects degraded (3.663%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T09:00:33.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:32 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3970952147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:33.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:32 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3970952147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:33.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:32 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3970952147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:33.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:00:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:00:33] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2024-04-12T09:00:34.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:33 smithi114 ceph-mon[89482]: pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 9132/246753 objects degraded (3.701%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T09:00:34.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:33 smithi090 ceph-mon[99575]: pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 9132/246753 objects degraded (3.701%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T09:00:34.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:33 smithi090 ceph-mon[95814]: pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 9132/246753 objects degraded (3.701%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T09:00:35.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:34 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/869851015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:35.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:34 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/869851015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:35.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:34 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/869851015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:35 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 9132/246753 objects degraded (3.701%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:35 smithi114 ceph-mon[89482]: pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 9132/245244 objects degraded (3.724%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T09:00:36.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:00:35 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:00:35.770+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:36.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:35 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 9132/246753 objects degraded (3.701%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:36.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:35 smithi090 ceph-mon[99575]: pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 9132/245244 objects degraded (3.724%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T09:00:36.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:35 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 9132/246753 objects degraded (3.701%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:36.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:35 smithi090 ceph-mon[95814]: pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 9132/245244 objects degraded (3.724%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T09:00:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:36 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:36 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:36 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:37.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:36 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:37.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:36 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:37.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:36 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:37.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:36 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:37.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:36 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:37.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:36 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:37 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3513000042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:37 smithi114 ceph-mon[89482]: pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 8961/237432 objects degraded (3.774%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T09:00:38.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:37 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3513000042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:38.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:37 smithi090 ceph-mon[95814]: pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 8961/237432 objects degraded (3.774%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T09:00:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:37 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3513000042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:38.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:37 smithi090 ceph-mon[99575]: pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 8961/237432 objects degraded (3.774%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T09:00:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:40 smithi090 ceph-mon[95814]: pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 8961/233859 objects degraded (3.832%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T09:00:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:40 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4097000845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:40 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 8961/233859 objects degraded (3.832%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:40 smithi090 ceph-mon[99575]: pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 8961/233859 objects degraded (3.832%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T09:00:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:40 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4097000845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:40 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 8961/233859 objects degraded (3.832%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:41.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:40 smithi114 ceph-mon[89482]: pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 8961/233859 objects degraded (3.832%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T09:00:41.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:40 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4097000845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:41.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:40 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 8961/233859 objects degraded (3.832%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:42.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:42 smithi090 ceph-mon[95814]: pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 8961/230973 objects degraded (3.880%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T09:00:42.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:42 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/545313112' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:42 smithi090 ceph-mon[99575]: pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 8961/230973 objects degraded (3.880%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T09:00:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:42 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/545313112' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:42 smithi114 ceph-mon[89482]: pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 8961/230973 objects degraded (3.880%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T09:00:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:42 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/545313112' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:43.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:00:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:00:43] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2024-04-12T09:00:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:44 smithi090 ceph-mon[95814]: pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 8775/223599 objects degraded (3.924%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T09:00:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:44 smithi090 ceph-mon[99575]: pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 8775/223599 objects degraded (3.924%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T09:00:45.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:44 smithi114 ceph-mon[89482]: pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 8775/223599 objects degraded (3.924%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T09:00:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:45 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2925630795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:45 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 8775/223599 objects degraded (3.924%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:45 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2925630795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:45 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 8775/223599 objects degraded (3.924%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:45 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2925630795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:45 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 8775/223599 objects degraded (3.924%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:00:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:46 smithi090 ceph-mon[99575]: pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 8775/222240 objects degraded (3.948%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T09:00:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:46 smithi090 ceph-mon[95814]: pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 8775/222240 objects degraded (3.948%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T09:00:47.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:46 smithi114 ceph-mon[89482]: pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 8775/222240 objects degraded (3.948%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T09:00:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:47 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2856626894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:47 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2856626894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:47 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2856626894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:48.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:48 smithi090 ceph-mon[99575]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 8591/214311 objects degraded (4.009%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T09:00:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:48 smithi090 ceph-mon[95814]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 8591/214311 objects degraded (4.009%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T09:00:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:48 smithi114 ceph-mon[89482]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 8591/214311 objects degraded (4.009%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T09:00:50.918 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:50 smithi114 ceph-mon[89482]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 8591/210531 objects degraded (4.081%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T09:00:50.919 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:50 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1115963220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:50.919 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:50 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 8591/210531 objects degraded (4.081%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:50.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:50 smithi090 ceph-mon[95814]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 8591/210531 objects degraded (4.081%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T09:00:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:50 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1115963220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:50 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 8591/210531 objects degraded (4.081%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:50.923 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:00:50 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:00:50.787+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:50 smithi090 ceph-mon[99575]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 8591/210531 objects degraded (4.081%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T09:00:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:50 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1115963220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:50.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:50 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 8591/210531 objects degraded (4.081%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:51.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:51 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:51.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:51 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:52.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:51 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:52.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:52 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:52.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:52 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:52.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:52 smithi090 ceph-mon[95814]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8591/207606 objects degraded (4.138%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T09:00:52.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:52 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:52.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:52 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:52.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:52 smithi090 ceph-mon[99575]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8591/207606 objects degraded (4.138%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T09:00:53.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:52 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:00:53.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:52 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T09:00:53.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:52 smithi114 ceph-mon[89482]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8591/207606 objects degraded (4.138%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T09:00:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:00:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:00:53] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2024-04-12T09:00:54.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:53 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3619560496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:54.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:53 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3619560496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:54.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:53 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3619560496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:55.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:54 smithi114 ceph-mon[89482]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 8387/200388 objects degraded (4.185%); 3.0 MiB/s, 47 objects/s recovering 2024-04-12T09:00:55.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:54 smithi090 ceph-mon[95814]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 8387/200388 objects degraded (4.185%); 3.0 MiB/s, 47 objects/s recovering 2024-04-12T09:00:55.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:54 smithi090 ceph-mon[99575]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 8387/200388 objects degraded (4.185%); 3.0 MiB/s, 47 objects/s recovering 2024-04-12T09:00:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:55 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2981195572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:55 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 8387/200388 objects degraded (4.185%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:55 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2981195572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:56.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:55 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 8387/200388 objects degraded (4.185%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:56.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:55 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2981195572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:56.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:55 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 8387/200388 objects degraded (4.185%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:00:57.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:56 smithi114 ceph-mon[89482]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 8387/198828 objects degraded (4.218%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T09:00:57.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:56 smithi090 ceph-mon[95814]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 8387/198828 objects degraded (4.218%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T09:00:57.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:56 smithi090 ceph-mon[99575]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 8387/198828 objects degraded (4.218%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T09:00:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:58 smithi114 ceph-mon[89482]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 8149/190716 objects degraded (4.273%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T09:00:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:00:58 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2071354662' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:59.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:58 smithi090 ceph-mon[95814]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 8149/190716 objects degraded (4.273%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T09:00:59.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:00:58 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2071354662' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:00:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:58 smithi090 ceph-mon[99575]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 8149/190716 objects degraded (4.273%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T09:00:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:00:58 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2071354662' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:01.015 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:00 smithi114 ceph-mon[89482]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 8149/188595 objects degraded (4.321%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T09:01:01.016 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:00 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4094449433' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:01.016 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:00 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 8149/188595 objects degraded (4.321%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:01.016 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:00 smithi090 ceph-mon[99575]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 8149/188595 objects degraded (4.321%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T09:01:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:00 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4094449433' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:00 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 8149/188595 objects degraded (4.321%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:00 smithi090 ceph-mon[95814]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 8149/188595 objects degraded (4.321%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T09:01:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:00 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4094449433' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:00 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 8149/188595 objects degraded (4.321%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:02.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:01 smithi114 ceph-mon[89482]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 8149/184290 objects degraded (4.422%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T09:01:02.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:01 smithi090 ceph-mon[99575]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 8149/184290 objects degraded (4.422%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T09:01:02.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:01 smithi090 ceph-mon[95814]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 8149/184290 objects degraded (4.422%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T09:01:03.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:03 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/192882242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:03.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:03 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/192882242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:03.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:01:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:01:03] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2024-04-12T09:01:03.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:03 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/192882242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:04.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:04 smithi114 ceph-mon[89482]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 7917/176820 objects degraded (4.477%); 3.5 MiB/s, 56 objects/s recovering 2024-04-12T09:01:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:04 smithi090 ceph-mon[95814]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 7917/176820 objects degraded (4.477%); 3.5 MiB/s, 56 objects/s recovering 2024-04-12T09:01:04.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:04 smithi090 ceph-mon[99575]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 7917/176820 objects degraded (4.477%); 3.5 MiB/s, 56 objects/s recovering 2024-04-12T09:01:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:05 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7917/176820 objects degraded (4.477%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:05.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:05 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7917/176820 objects degraded (4.477%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:05 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7917/176820 objects degraded (4.477%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:06 smithi114 ceph-mon[89482]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 7917/175413 objects degraded (4.513%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T09:01:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:06 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/655339664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:06 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:01:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:06 smithi090 ceph-mon[99575]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 7917/175413 objects degraded (4.513%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T09:01:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:06 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/655339664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:06.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:06 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:01:06.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:06 smithi090 ceph-mon[95814]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 7917/175413 objects degraded (4.513%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T09:01:06.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:06 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/655339664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:06.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:06 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:01:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:08 smithi114 ceph-mon[89482]: osdmap e123: 8 total, 8 up, 8 in 2024-04-12T09:01:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:08 smithi114 ceph-mon[89482]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 7663/167418 objects degraded (4.577%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T09:01:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:08 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/652156226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:08.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:08 smithi090 ceph-mon[95814]: osdmap e123: 8 total, 8 up, 8 in 2024-04-12T09:01:08.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:08 smithi090 ceph-mon[95814]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 7663/167418 objects degraded (4.577%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T09:01:08.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:08 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/652156226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:08.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:08 smithi090 ceph-mon[99575]: osdmap e123: 8 total, 8 up, 8 in 2024-04-12T09:01:08.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:08 smithi090 ceph-mon[99575]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 7663/167418 objects degraded (4.577%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T09:01:08.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:08 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/652156226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:09.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:09 smithi090 ceph-mon[99575]: osdmap e124: 8 total, 8 up, 8 in 2024-04-12T09:01:09.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:09 smithi090 ceph-mon[95814]: osdmap e124: 8 total, 8 up, 8 in 2024-04-12T09:01:09.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:09 smithi114 ceph-mon[89482]: osdmap e124: 8 total, 8 up, 8 in 2024-04-12T09:01:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:10 smithi114 ceph-mon[89482]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 7663/165264 objects degraded (4.637%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T09:01:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:10 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:10 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:10 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7663/167418 objects degraded (4.577%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:10.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:10 smithi090 ceph-mon[95814]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 7663/165264 objects degraded (4.637%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T09:01:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:10 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:10 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:10 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7663/167418 objects degraded (4.577%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:10 smithi090 ceph-mon[99575]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 7663/165264 objects degraded (4.637%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T09:01:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:10 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:10 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:10 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7663/167418 objects degraded (4.577%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:11 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1457569084' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:11.880 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:11 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1457569084' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:11.881 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:11 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1457569084' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:12 smithi114 ceph-mon[89482]: pgmap v392: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 7347/162375 objects degraded (4.525%); 4.4 MiB/s, 71 objects/s recovering 2024-04-12T09:01:12.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:12 smithi090 ceph-mon[99575]: pgmap v392: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 7347/162375 objects degraded (4.525%); 4.4 MiB/s, 71 objects/s recovering 2024-04-12T09:01:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:12 smithi090 ceph-mon[95814]: pgmap v392: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 7347/162375 objects degraded (4.525%); 4.4 MiB/s, 71 objects/s recovering 2024-04-12T09:01:13.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:01:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:01:13] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2024-04-12T09:01:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:13 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:13 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:13 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4293029718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:13 smithi090 ceph-mon[99575]: pgmap v393: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7347/157896 objects degraded (4.653%); 4.4 MiB/s, 71 objects/s recovering 2024-04-12T09:01:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:14.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:13 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4293029718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:14.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:13 smithi090 ceph-mon[95814]: pgmap v393: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7347/157896 objects degraded (4.653%); 4.4 MiB/s, 71 objects/s recovering 2024-04-12T09:01:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:13 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:13 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:13 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4293029718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:13 smithi114 ceph-mon[89482]: pgmap v393: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7347/157896 objects degraded (4.653%); 4.4 MiB/s, 71 objects/s recovering 2024-04-12T09:01:14.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:01:14.250+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: pgmap v394: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 7347/157896 objects degraded (4.653%); 3.1 MiB/s, 48 objects/s recovering 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: pgmap v395: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 7347/157896 objects degraded (4.653%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:15.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:14 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:15.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:01:15.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:01:15.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: pgmap v394: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 7347/157896 objects degraded (4.653%); 3.1 MiB/s, 48 objects/s recovering 2024-04-12T09:01:15.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: pgmap v395: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 7347/157896 objects degraded (4.653%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T09:01:15.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:15.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:01:15.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:01:15.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: pgmap v394: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 7347/157896 objects degraded (4.653%); 3.1 MiB/s, 48 objects/s recovering 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: pgmap v395: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 7347/157896 objects degraded (4.653%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:01:15.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:01:15.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:01:15.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:01:15.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:15.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:15.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:14 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:15 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7347/157896 objects degraded (4.653%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:15 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:15 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/731692883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:15 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7347/157896 objects degraded (4.653%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:15 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:16.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:15 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/731692883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:16.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:15 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7347/157896 objects degraded (4.653%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:16.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:16.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:15 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/731692883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:17.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:16 smithi114 ceph-mon[89482]: pgmap v396: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 7347/155829 objects degraded (4.715%); 2.6 MiB/s, 42 objects/s recovering 2024-04-12T09:01:17.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:16 smithi090 ceph-mon[99575]: pgmap v396: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 7347/155829 objects degraded (4.715%); 2.6 MiB/s, 42 objects/s recovering 2024-04-12T09:01:17.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:16 smithi090 ceph-mon[95814]: pgmap v396: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 7347/155829 objects degraded (4.715%); 2.6 MiB/s, 42 objects/s recovering 2024-04-12T09:01:19.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:19 smithi114 ceph-mon[89482]: pgmap v397: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 7170/146778 objects degraded (4.885%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T09:01:19.370 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:19 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1963581538' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:19.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:19 smithi090 ceph-mon[99575]: pgmap v397: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 7170/146778 objects degraded (4.885%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T09:01:19.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:19 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1963581538' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:19.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:19 smithi090 ceph-mon[95814]: pgmap v397: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 7170/146778 objects degraded (4.885%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T09:01:19.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:19 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1963581538' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:21.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:21 smithi114 ceph-mon[89482]: pgmap v398: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 7170/144480 objects degraded (4.963%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T09:01:21.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:21 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 7170/144480 objects degraded (4.963%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:21.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:21 smithi090 ceph-mon[99575]: pgmap v398: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 7170/144480 objects degraded (4.963%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T09:01:21.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:21 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 7170/144480 objects degraded (4.963%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:21.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:21 smithi090 ceph-mon[95814]: pgmap v398: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 7170/144480 objects degraded (4.963%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T09:01:21.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:21 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 7170/144480 objects degraded (4.963%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:22.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:22 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/309146546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:22.458 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:22 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/309146546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:22.459 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:22 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/309146546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:23.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:23 smithi114 ceph-mon[89482]: pgmap v399: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 6955/136980 objects degraded (5.077%); 2.9 MiB/s, 46 objects/s recovering 2024-04-12T09:01:23.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:01:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:01:23] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2024-04-12T09:01:23.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:23 smithi090 ceph-mon[99575]: pgmap v399: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 6955/136980 objects degraded (5.077%); 2.9 MiB/s, 46 objects/s recovering 2024-04-12T09:01:23.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:23 smithi090 ceph-mon[95814]: pgmap v399: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 6955/136980 objects degraded (5.077%); 2.9 MiB/s, 46 objects/s recovering 2024-04-12T09:01:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:24 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3727210284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:24.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:24 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3727210284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:24.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:24 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3727210284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:25.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:25 smithi090 ceph-mon[95814]: pgmap v400: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 6955/134706 objects degraded (5.163%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T09:01:25.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:25 smithi090 ceph-mon[99575]: pgmap v400: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 6955/134706 objects degraded (5.163%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T09:01:25.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:25 smithi114 ceph-mon[89482]: pgmap v400: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 6955/134706 objects degraded (5.163%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T09:01:26.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:26 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 6955/134706 objects degraded (5.163%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:26 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 6955/134706 objects degraded (5.163%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:26.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:26 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 6955/134706 objects degraded (5.163%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:27.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:27 smithi090 ceph-mon[95814]: pgmap v401: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 6955/132642 objects degraded (5.243%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T09:01:27.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:27 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2931518160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:27.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:27 smithi090 ceph-mon[99575]: pgmap v401: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 6955/132642 objects degraded (5.243%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T09:01:27.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:27 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2931518160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:27.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:27 smithi114 ceph-mon[89482]: pgmap v401: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 6955/132642 objects degraded (5.243%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T09:01:27.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:27 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2931518160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:29.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:29 smithi090 ceph-mon[95814]: pgmap v402: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 6749/123057 objects degraded (5.484%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T09:01:29.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:29 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3523637417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:29.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:01:29 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:01:29.268+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:29.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:29 smithi090 ceph-mon[99575]: pgmap v402: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 6749/123057 objects degraded (5.484%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T09:01:29.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:29 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3523637417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:29.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:29 smithi114 ceph-mon[89482]: pgmap v402: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 6749/123057 objects degraded (5.484%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T09:01:29.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:29 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3523637417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:30.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:30 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:30.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:30 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:30.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:30 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:30.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:30 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:30.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:30 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:30.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:30 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:30.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:30 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:30.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:30 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:30.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:30 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:31 smithi114 ceph-mon[89482]: pgmap v403: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 6749/120912 objects degraded (5.582%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T09:01:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:31 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 6749/120912 objects degraded (5.582%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:31 smithi090 ceph-mon[95814]: pgmap v403: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 6749/120912 objects degraded (5.582%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T09:01:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:31 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 6749/120912 objects degraded (5.582%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:31 smithi090 ceph-mon[99575]: pgmap v403: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 6749/120912 objects degraded (5.582%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T09:01:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:31 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 6749/120912 objects degraded (5.582%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:32.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:32 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2626498411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:32.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:32 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2626498411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:32 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2626498411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:33.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:33 smithi114 ceph-mon[89482]: pgmap v404: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 6531/113058 objects degraded (5.777%); 3.3 MiB/s, 53 objects/s recovering 2024-04-12T09:01:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:01:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:01:33] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2024-04-12T09:01:33.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:33 smithi090 ceph-mon[95814]: pgmap v404: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 6531/113058 objects degraded (5.777%); 3.3 MiB/s, 53 objects/s recovering 2024-04-12T09:01:33.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:33 smithi090 ceph-mon[99575]: pgmap v404: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 6531/113058 objects degraded (5.777%); 3.3 MiB/s, 53 objects/s recovering 2024-04-12T09:01:34.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:34 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/397406148' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:34.472 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:34 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/397406148' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:34.473 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:34 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/397406148' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:35 smithi114 ceph-mon[89482]: pgmap v405: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 6531/110709 objects degraded (5.899%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T09:01:35.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:35 smithi090 ceph-mon[95814]: pgmap v405: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 6531/110709 objects degraded (5.899%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T09:01:35.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:35 smithi090 ceph-mon[99575]: pgmap v405: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 6531/110709 objects degraded (5.899%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T09:01:36.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:36 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 6531/110709 objects degraded (5.899%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:36.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:36 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 6531/110709 objects degraded (5.899%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:36.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:36 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 6531/110709 objects degraded (5.899%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:37.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:37 smithi114 ceph-mon[89482]: pgmap v406: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6531/108477 objects degraded (6.021%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T09:01:37.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:37 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/886131344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:37.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:37 smithi090 ceph-mon[95814]: pgmap v406: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6531/108477 objects degraded (6.021%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T09:01:37.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:37 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/886131344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:37.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:37 smithi090 ceph-mon[99575]: pgmap v406: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6531/108477 objects degraded (6.021%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T09:01:37.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:37 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/886131344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:39.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:39 smithi114 ceph-mon[89482]: pgmap v407: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 6262/98898 objects degraded (6.332%); 2.2 MiB/s, 57 objects/s recovering 2024-04-12T09:01:39.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:39 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1116624110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:39.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:39 smithi090 ceph-mon[95814]: pgmap v407: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 6262/98898 objects degraded (6.332%); 2.2 MiB/s, 57 objects/s recovering 2024-04-12T09:01:39.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:39 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1116624110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:39.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:39 smithi090 ceph-mon[99575]: pgmap v407: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 6262/98898 objects degraded (6.332%); 2.2 MiB/s, 57 objects/s recovering 2024-04-12T09:01:39.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:39 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1116624110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:41.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:41 smithi114 ceph-mon[89482]: pgmap v408: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 6262/96606 objects degraded (6.482%); 1.2 MiB/s, 40 objects/s recovering 2024-04-12T09:01:41.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:41 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 6262/96606 objects degraded (6.482%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:41 smithi090 ceph-mon[95814]: pgmap v408: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 6262/96606 objects degraded (6.482%); 1.2 MiB/s, 40 objects/s recovering 2024-04-12T09:01:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:41 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 6262/96606 objects degraded (6.482%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:41.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:41 smithi090 ceph-mon[99575]: pgmap v408: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 6262/96606 objects degraded (6.482%); 1.2 MiB/s, 40 objects/s recovering 2024-04-12T09:01:41.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:41 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 6262/96606 objects degraded (6.482%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:42.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:42 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2051009755' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:42.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:42 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2051009755' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:42.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:42 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2051009755' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:43 smithi114 ceph-mon[89482]: pgmap v409: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5974/88896 objects degraded (6.720%); 1.2 MiB/s, 64 objects/s recovering 2024-04-12T09:01:43.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:43 smithi090 ceph-mon[95814]: pgmap v409: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5974/88896 objects degraded (6.720%); 1.2 MiB/s, 64 objects/s recovering 2024-04-12T09:01:43.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:01:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:01:43] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2024-04-12T09:01:43.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:43 smithi090 ceph-mon[99575]: pgmap v409: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5974/88896 objects degraded (6.720%); 1.2 MiB/s, 64 objects/s recovering 2024-04-12T09:01:44.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:01:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:01:44.285+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:45 smithi114 ceph-mon[89482]: pgmap v410: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 5974/86574 objects degraded (6.900%); 37 KiB/s, 46 objects/s recovering 2024-04-12T09:01:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:45 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:45 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:45 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3450060246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:45.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:45 smithi090 ceph-mon[99575]: pgmap v410: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 5974/86574 objects degraded (6.900%); 37 KiB/s, 46 objects/s recovering 2024-04-12T09:01:45.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:45.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:45 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:45.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:45 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:45.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:45 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3450060246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:45 smithi090 ceph-mon[95814]: pgmap v410: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 5974/86574 objects degraded (6.900%); 37 KiB/s, 46 objects/s recovering 2024-04-12T09:01:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:45 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:45 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T09:01:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:45 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3450060246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:46.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:46 smithi114 ceph-mon[89482]: osdmap e125: 8 total, 8 up, 8 in 2024-04-12T09:01:46.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:46 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 5974/86574 objects degraded (6.900%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:46.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:46 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:46.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:46 smithi090 ceph-mon[99575]: osdmap e125: 8 total, 8 up, 8 in 2024-04-12T09:01:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:46 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 5974/86574 objects degraded (6.900%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:46 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:46.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:46 smithi090 ceph-mon[95814]: osdmap e125: 8 total, 8 up, 8 in 2024-04-12T09:01:46.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:46 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 5974/86574 objects degraded (6.900%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:46.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:46 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:01:47.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:47 smithi114 ceph-mon[89482]: pgmap v412: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 5974/84321 objects degraded (7.085%); 45 KiB/s, 55 objects/s recovering 2024-04-12T09:01:47.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:47 smithi114 ceph-mon[89482]: osdmap e126: 8 total, 8 up, 8 in 2024-04-12T09:01:47.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:47 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1204103881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:47.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:47 smithi090 ceph-mon[99575]: pgmap v412: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 5974/84321 objects degraded (7.085%); 45 KiB/s, 55 objects/s recovering 2024-04-12T09:01:47.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:47 smithi090 ceph-mon[99575]: osdmap e126: 8 total, 8 up, 8 in 2024-04-12T09:01:47.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:47 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1204103881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:47.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:47 smithi090 ceph-mon[95814]: pgmap v412: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 5974/84321 objects degraded (7.085%); 45 KiB/s, 55 objects/s recovering 2024-04-12T09:01:47.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:47 smithi090 ceph-mon[95814]: osdmap e126: 8 total, 8 up, 8 in 2024-04-12T09:01:47.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:47 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1204103881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:49.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:49 smithi114 ceph-mon[89482]: pgmap v414: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5763/76302 objects degraded (7.553%); 0 B/s, 62 objects/s recovering 2024-04-12T09:01:49.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:49 smithi090 ceph-mon[99575]: pgmap v414: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5763/76302 objects degraded (7.553%); 0 B/s, 62 objects/s recovering 2024-04-12T09:01:49.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:49 smithi090 ceph-mon[95814]: pgmap v414: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5763/76302 objects degraded (7.553%); 0 B/s, 62 objects/s recovering 2024-04-12T09:01:50.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:50 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3723496845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:50.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:50 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3723496845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:50.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:50 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3723496845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:51.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:51 smithi114 ceph-mon[89482]: pgmap v415: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 5514/73221 objects degraded (7.531%); 0 B/s, 57 objects/s recovering 2024-04-12T09:01:51.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:51 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 5514/73221 objects degraded (7.531%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:51.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:51 smithi090 ceph-mon[95814]: pgmap v415: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 5514/73221 objects degraded (7.531%); 0 B/s, 57 objects/s recovering 2024-04-12T09:01:51.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:51 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 5514/73221 objects degraded (7.531%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:51.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:51 smithi090 ceph-mon[99575]: pgmap v415: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 5514/73221 objects degraded (7.531%); 0 B/s, 57 objects/s recovering 2024-04-12T09:01:51.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:51 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 5514/73221 objects degraded (7.531%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:52.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:52 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2848637212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:52.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:52 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2848637212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:52.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:52 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2848637212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:53.574 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:53 smithi090 ceph-mon[95814]: pgmap v416: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5514/66273 objects degraded (8.320%); 0 B/s, 57 objects/s recovering 2024-04-12T09:01:53.574 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:01:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:01:53] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2024-04-12T09:01:53.575 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:53 smithi090 ceph-mon[99575]: pgmap v416: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5514/66273 objects degraded (8.320%); 0 B/s, 57 objects/s recovering 2024-04-12T09:01:53.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:53 smithi114 ceph-mon[89482]: pgmap v416: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5514/66273 objects degraded (8.320%); 0 B/s, 57 objects/s recovering 2024-04-12T09:01:55.586 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:55 smithi090 ceph-mon[99575]: pgmap v417: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5342/63930 objects degraded (8.356%); 0 B/s, 71 objects/s recovering 2024-04-12T09:01:55.586 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:55 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1434317392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:55.586 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:55 smithi090 ceph-mon[95814]: pgmap v417: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5342/63930 objects degraded (8.356%); 0 B/s, 71 objects/s recovering 2024-04-12T09:01:55.586 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:55 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1434317392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:55.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:55 smithi114 ceph-mon[89482]: pgmap v417: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5342/63930 objects degraded (8.356%); 0 B/s, 71 objects/s recovering 2024-04-12T09:01:55.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:55 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1434317392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:56.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:56 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 5342/63930 objects degraded (8.356%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:56.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:56 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 5342/63930 objects degraded (8.356%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:56.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:56 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 5342/63930 objects degraded (8.356%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:01:57.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:57 smithi114 ceph-mon[89482]: pgmap v418: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 5342/60897 objects degraded (8.772%); 0 B/s, 63 objects/s recovering 2024-04-12T09:01:57.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:57 smithi090 ceph-mon[99575]: pgmap v418: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 5342/60897 objects degraded (8.772%); 0 B/s, 63 objects/s recovering 2024-04-12T09:01:57.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:57 smithi090 ceph-mon[95814]: pgmap v418: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 5342/60897 objects degraded (8.772%); 0 B/s, 63 objects/s recovering 2024-04-12T09:01:58.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:58 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3880215967' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:58.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:58 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3880215967' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:58.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:58 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3880215967' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:01:59.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:59 smithi114 ceph-mon[89482]: pgmap v419: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 5167/52257 objects degraded (9.888%); 0 B/s, 68 objects/s recovering 2024-04-12T09:01:59.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:01:59 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:59.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:59 smithi090 ceph-mon[95814]: pgmap v419: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 5167/52257 objects degraded (9.888%); 0 B/s, 68 objects/s recovering 2024-04-12T09:01:59.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:01:59 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:01:59.674 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:01:59 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:01:59.304+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T09:01:59.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:59 smithi090 ceph-mon[99575]: pgmap v419: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 5167/52257 objects degraded (9.888%); 0 B/s, 68 objects/s recovering 2024-04-12T09:01:59.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:01:59 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:00.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:00 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:00.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:00 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T09:02:00.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:00 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/657963686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:00 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:00 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T09:02:00.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:00 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/657963686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:00.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:00 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:00.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:00 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T09:02:00.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:00 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/657963686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:01.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:01 smithi114 ceph-mon[89482]: pgmap v420: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 5167/49233 objects degraded (10.495%); 0 B/s, 49 objects/s recovering 2024-04-12T09:02:01.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:01 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 5167/49233 objects degraded (10.495%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:01.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:01 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:01.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:01 smithi090 ceph-mon[99575]: pgmap v420: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 5167/49233 objects degraded (10.495%); 0 B/s, 49 objects/s recovering 2024-04-12T09:02:01.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:01 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 5167/49233 objects degraded (10.495%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:01.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:01.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:01 smithi090 ceph-mon[95814]: pgmap v420: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 5167/49233 objects degraded (10.495%); 0 B/s, 49 objects/s recovering 2024-04-12T09:02:01.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:01 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 5167/49233 objects degraded (10.495%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:01.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:03.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:03 smithi114 ceph-mon[89482]: pgmap v421: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 880 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 5167/42309 objects degraded (12.213%); 0 B/s, 28 objects/s recovering 2024-04-12T09:02:03.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:03 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2206318691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:03.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:02:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:02:03] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2024-04-12T09:02:03.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:03 smithi090 ceph-mon[99575]: pgmap v421: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 880 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 5167/42309 objects degraded (12.213%); 0 B/s, 28 objects/s recovering 2024-04-12T09:02:03.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:03 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2206318691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:03.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:03 smithi090 ceph-mon[95814]: pgmap v421: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 880 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 5167/42309 objects degraded (12.213%); 0 B/s, 28 objects/s recovering 2024-04-12T09:02:03.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:03 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2206318691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:05.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:05 smithi090 ceph-mon[99575]: pgmap v422: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 834 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 4962/40056 objects degraded (12.388%); 0 B/s, 45 objects/s recovering 2024-04-12T09:02:05.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:05 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4241342398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:05.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:05 smithi090 ceph-mon[95814]: pgmap v422: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 834 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 4962/40056 objects degraded (12.388%); 0 B/s, 45 objects/s recovering 2024-04-12T09:02:05.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:05 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4241342398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:05 smithi114 ceph-mon[89482]: pgmap v422: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 834 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 4962/40056 objects degraded (12.388%); 0 B/s, 45 objects/s recovering 2024-04-12T09:02:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:05 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4241342398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:06.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:06 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 4962/40056 objects degraded (12.388%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:06.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:06 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 4962/40056 objects degraded (12.388%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:06 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 4962/40056 objects degraded (12.388%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:07 smithi090 ceph-mon[99575]: pgmap v423: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 772 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 4962/37101 objects degraded (13.374%); 0 B/s, 31 objects/s recovering 2024-04-12T09:02:07.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:07 smithi090 ceph-mon[95814]: pgmap v423: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 772 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 4962/37101 objects degraded (13.374%); 0 B/s, 31 objects/s recovering 2024-04-12T09:02:07.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:07 smithi114 ceph-mon[89482]: pgmap v423: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 772 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 4962/37101 objects degraded (13.374%); 0 B/s, 31 objects/s recovering 2024-04-12T09:02:08.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:08 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1685079602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:08.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:08 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1685079602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:08 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1685079602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:09.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:09 smithi090 ceph-mon[95814]: pgmap v424: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 578 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 4766/27771 objects degraded (17.162%); 0 B/s, 47 objects/s recovering 2024-04-12T09:02:09.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:09 smithi090 ceph-mon[99575]: pgmap v424: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 578 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 4766/27771 objects degraded (17.162%); 0 B/s, 47 objects/s recovering 2024-04-12T09:02:09.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:09 smithi114 ceph-mon[89482]: pgmap v424: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 578 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 4766/27771 objects degraded (17.162%); 0 B/s, 47 objects/s recovering 2024-04-12T09:02:10.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:10 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1494872402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:10.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:10 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1494872402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:10 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1494872402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:11.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:11 smithi090 ceph-mon[95814]: pgmap v425: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 512 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 4766/24612 objects degraded (19.365%); 0 B/s, 33 objects/s recovering 2024-04-12T09:02:11.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:11 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 4766/24612 objects degraded (19.365%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:11 smithi090 ceph-mon[99575]: pgmap v425: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 512 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 4766/24612 objects degraded (19.365%); 0 B/s, 33 objects/s recovering 2024-04-12T09:02:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:11 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 4766/24612 objects degraded (19.365%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:11 smithi114 ceph-mon[89482]: pgmap v425: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 512 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 4766/24612 objects degraded (19.365%); 0 B/s, 33 objects/s recovering 2024-04-12T09:02:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:11 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 4766/24612 objects degraded (19.365%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:13.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:13 smithi090 ceph-mon[95814]: pgmap v426: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 369 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 4766/17748 objects degraded (26.854%); 0 B/s, 33 objects/s recovering 2024-04-12T09:02:13.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:13 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1421767913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:02:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:02:13] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2024-04-12T09:02:13.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:13 smithi090 ceph-mon[99575]: pgmap v426: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 369 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 4766/17748 objects degraded (26.854%); 0 B/s, 33 objects/s recovering 2024-04-12T09:02:13.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:13 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1421767913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:13 smithi114 ceph-mon[89482]: pgmap v426: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 369 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 4766/17748 objects degraded (26.854%); 0 B/s, 33 objects/s recovering 2024-04-12T09:02:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:13 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1421767913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:14.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:14 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:14.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:14 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:14.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:14 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:15.723 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:15 smithi090 ceph-mon[99575]: pgmap v427: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 318 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 4552/15321 objects degraded (29.711%); 0 B/s, 51 objects/s recovering 2024-04-12T09:02:15.724 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:15 smithi090 ceph-mon[95814]: pgmap v427: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 318 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 4552/15321 objects degraded (29.711%); 0 B/s, 51 objects/s recovering 2024-04-12T09:02:15.777 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:15 smithi114 ceph-mon[89482]: pgmap v427: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 318 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 4552/15321 objects degraded (29.711%); 0 B/s, 51 objects/s recovering 2024-04-12T09:02:16.747 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:16 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:16.747 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:16 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 4552/15321 objects degraded (29.711%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:16.747 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:16 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3976843512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:16 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:16 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 4552/15321 objects degraded (29.711%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:16 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3976843512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:16 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 4552/15321 objects degraded (29.711%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:16 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3976843512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:17.758 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:17 smithi114 ceph-mon[89482]: pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 254 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 4552/12246 objects degraded (37.171%); 0 B/s, 34 objects/s recovering 2024-04-12T09:02:17.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:17 smithi090 ceph-mon[95814]: pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 254 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 4552/12246 objects degraded (37.171%); 0 B/s, 34 objects/s recovering 2024-04-12T09:02:17.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:17 smithi090 ceph-mon[99575]: pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 254 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 4552/12246 objects degraded (37.171%); 0 B/s, 34 objects/s recovering 2024-04-12T09:02:17.994 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Removed 121923 objects 2024-04-12T09:02:17.994 INFO:tasks.radosbench.radosbench.0.smithi090.stdout:Clean up completed and total clean up time :158.105 2024-04-12T09:02:18.746 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:18 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/14649043' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:18.801 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:18 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/14649043' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:18.801 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:18 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/14649043' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:18.978 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-04-12T09:02:18.978 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-04-12T09:02:19.625 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:02:19.752 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:19 smithi090 ceph-mon[99575]: pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 74 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 4355/3591 objects degraded (121.275%); 0 B/s, 50 objects/s recovering 2024-04-12T09:02:19.752 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:19 smithi090 ceph-mon[95814]: pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 74 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 4355/3591 objects degraded (121.275%); 0 B/s, 50 objects/s recovering 2024-04-12T09:02:19.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:19 smithi114 ceph-mon[89482]: pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 74 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 4355/3591 objects degraded (121.275%); 0 B/s, 50 objects/s recovering 2024-04-12T09:02:20.780 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[99575]: osdmap e127: 8 total, 8 up, 8 in 2024-04-12T09:02:20.780 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.781 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.781 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.781 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.781 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2267804640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:20.781 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[95814]: osdmap e127: 8 total, 8 up, 8 in 2024-04-12T09:02:20.781 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.781 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.781 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.781 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.781 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2267804640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:20 smithi114 ceph-mon[89482]: osdmap e127: 8 total, 8 up, 8 in 2024-04-12T09:02:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:20 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:20 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:20 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:20 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:20.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:20 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2267804640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:21.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:02:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:02:20.778+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T09:02:21.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 37 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 4355/1812 objects degraded (240.342%); 0 B/s, 41 objects/s recovering 2024-04-12T09:02:21.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: osdmap e128: 8 total, 8 up, 8 in 2024-04-12T09:02:21.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 4355/3591 objects degraded (121.275%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:21.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:02:21.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:02:21.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: pgmap v433: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 37 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4355/1812 objects degraded (240.342%); 0 B/s, 29 objects/s recovering 2024-04-12T09:02:21.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: pgmap v434: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 37 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 4355/1812 objects degraded (240.342%); 0 B/s, 43 objects/s recovering 2024-04-12T09:02:21.769 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:21.770 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:21.770 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:21.770 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:21.770 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:21.770 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:21 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:21.805 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 37 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 4355/1812 objects degraded (240.342%); 0 B/s, 41 objects/s recovering 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: osdmap e128: 8 total, 8 up, 8 in 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 4355/3591 objects degraded (121.275%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: pgmap v433: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 37 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4355/1812 objects degraded (240.342%); 0 B/s, 29 objects/s recovering 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: pgmap v434: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 37 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 4355/1812 objects degraded (240.342%); 0 B/s, 43 objects/s recovering 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:21.806 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 37 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 4355/1812 objects degraded (240.342%); 0 B/s, 41 objects/s recovering 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: osdmap e128: 8 total, 8 up, 8 in 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 4355/3591 objects degraded (121.275%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: pgmap v433: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 37 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4355/1812 objects degraded (240.342%); 0 B/s, 29 objects/s recovering 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: pgmap v434: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 37 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 4355/1812 objects degraded (240.342%); 0 B/s, 43 objects/s recovering 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:21.807 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:21 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:22.516 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'unique_pool_3' removed 2024-04-12T09:02:22.865 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:22 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:22.865 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:22 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T09:02:22.865 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:22 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1796247510' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T09:02:22.865 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:22 smithi090 ceph-mon[99575]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T09:02:22.865 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:22 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:22.865 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:22 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T09:02:22.866 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:22 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1796247510' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T09:02:22.866 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:22 smithi090 ceph-mon[95814]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T09:02:22.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:22 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:22.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:22 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-12T09:02:22.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:22 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1796247510' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T09:02:22.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:22 smithi114 ceph-mon[89482]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T09:02:23.257 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-12T09:02:23.258 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-04-12T09:02:23.258 DEBUG:teuthology.parallel:result is None 2024-04-12T09:02:23.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:02:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:02:23] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2024-04-12T09:02:23.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:23 smithi114 ceph-mon[89482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T09:02:23.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:23 smithi114 ceph-mon[89482]: osdmap e129: 8 total, 8 up, 8 in 2024-04-12T09:02:23.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:23 smithi114 ceph-mon[89482]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:23.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:23 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1183966748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:23 smithi090 ceph-mon[99575]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T09:02:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:23 smithi090 ceph-mon[99575]: osdmap e129: 8 total, 8 up, 8 in 2024-04-12T09:02:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:23 smithi090 ceph-mon[99575]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:23.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:23 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1183966748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:23.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:23 smithi090 ceph-mon[95814]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T09:02:23.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:23 smithi090 ceph-mon[95814]: osdmap e129: 8 total, 8 up, 8 in 2024-04-12T09:02:23.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:23 smithi090 ceph-mon[95814]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:23.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:23 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1183966748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:24.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:24 smithi090 ceph-mon[99575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4355/1812 objects degraded (240.342%), 3 pgs degraded, 3 pgs undersized) 2024-04-12T09:02:24.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:24 smithi090 ceph-mon[95814]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4355/1812 objects degraded (240.342%), 3 pgs degraded, 3 pgs undersized) 2024-04-12T09:02:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:24 smithi114 ceph-mon[89482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4355/1812 objects degraded (240.342%), 3 pgs degraded, 3 pgs undersized) 2024-04-12T09:02:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:25 smithi114 ceph-mon[89482]: pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2596787956' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:25.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:25 smithi090 ceph-mon[95814]: pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:25.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2596787956' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:25.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:25 smithi090 ceph-mon[99575]: pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:25.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2596787956' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:27 smithi114 ceph-mon[89482]: pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:28.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:27 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4183894179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:28.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:27 smithi090 ceph-mon[99575]: pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:28.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:27 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4183894179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:27 smithi090 ceph-mon[95814]: pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:27 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4183894179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:30.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:29 smithi114 ceph-mon[89482]: pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:30.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:29 smithi090 ceph-mon[99575]: pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:30.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:29 smithi090 ceph-mon[95814]: pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:30 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2367657793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:30 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:30 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:31.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:30 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2367657793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:31.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:30 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:31.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:30 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:31.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:30 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2367657793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:31.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:30 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:31.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:30 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:32.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:31 smithi114 ceph-mon[89482]: pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:32.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:31 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:32.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:31 smithi090 ceph-mon[95814]: pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:32.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:31 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:31 smithi090 ceph-mon[99575]: pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:32.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:31 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:32 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/104943406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:33.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:32 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/104943406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:33.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:32 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/104943406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:02:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:02:33] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-12T09:02:34.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:33 smithi114 ceph-mon[89482]: pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:34.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:33 smithi090 ceph-mon[95814]: pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:34.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:33 smithi090 ceph-mon[99575]: pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:35.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:34 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3400722384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:35.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:34 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3400722384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:35.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:34 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3400722384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:36.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:35 smithi114 ceph-mon[89482]: pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:36.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:35 smithi090 ceph-mon[99575]: pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:36.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:35 smithi090 ceph-mon[95814]: pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:36 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:37.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:36 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:37.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:36 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:37 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:37 smithi114 ceph-mon[89482]: Upgrade: osd.2 is safe to restart 2024-04-12T09:02:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:37 smithi114 ceph-mon[89482]: pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:37 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1587265742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:37 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:37 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T09:02:38.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:37 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:02:38.124 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:38.124 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[99575]: Upgrade: osd.2 is safe to restart 2024-04-12T09:02:38.124 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[99575]: pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:38.124 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1587265742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:38.124 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:38.124 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T09:02:38.124 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:02:38.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T09:02:38.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[95814]: Upgrade: osd.2 is safe to restart 2024-04-12T09:02:38.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[95814]: pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:38.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1587265742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:38.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:38.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T09:02:38.125 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:37 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:02:39.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:38 smithi114 ceph-mon[89482]: Upgrade: Updating osd.2 2024-04-12T09:02:39.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:38 smithi114 ceph-mon[89482]: Deploying daemon osd.2 on smithi090 2024-04-12T09:02:39.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:38 smithi090 ceph-mon[99575]: Upgrade: Updating osd.2 2024-04-12T09:02:39.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:38 smithi090 ceph-mon[99575]: Deploying daemon osd.2 on smithi090 2024-04-12T09:02:39.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:38 smithi090 ceph-mon[95814]: Upgrade: Updating osd.2 2024-04-12T09:02:39.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:38 smithi090 ceph-mon[95814]: Deploying daemon osd.2 on smithi090 2024-04-12T09:02:39.871 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:39 smithi090 ceph-mon[99575]: pgmap v444: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:39.872 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:39 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1713984231' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:39.872 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:39 smithi090 ceph-mon[95814]: pgmap v444: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:39.872 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:39 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1713984231' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:40.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:39 smithi114 ceph-mon[89482]: pgmap v444: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:40.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:39 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1713984231' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:41.984 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:41 smithi090 ceph-mon[99575]: pgmap v445: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:41.984 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:41 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1663559487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:41.985 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:41 smithi090 ceph-mon[95814]: pgmap v445: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:41.985 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:41 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1663559487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:41.985 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:41 smithi090 systemd[1]: Stopping Ceph osd.2 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:02:42.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:41 smithi114 ceph-mon[89482]: pgmap v445: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:42.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:41 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1663559487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:42.423 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[55718]: 2024-04-12T09:02:41.982+0000 7ff83c08f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:02:42.423 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[55718]: 2024-04-12T09:02:41.982+0000 7ff83c08f700 -1 osd.2 129 *** Got signal Terminated *** 2024-04-12T09:02:42.423 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[55718]: 2024-04-12T09:02:41.982+0000 7ff83c08f700 -1 osd.2 129 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T09:02:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:42 smithi114 ceph-mon[89482]: osd.2 marked itself down and dead 2024-04-12T09:02:43.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:42 smithi090 ceph-mon[99575]: osd.2 marked itself down and dead 2024-04-12T09:02:43.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:42 smithi090 ceph-mon[95814]: osd.2 marked itself down and dead 2024-04-12T09:02:43.477 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:02:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:02:43] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-12T09:02:43.477 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:43 smithi090 podman[125154]: 2024-04-12 09:02:43.19691017 +0000 UTC m=+1.333617952 container died 8008248b4e61ad0e130e68a9c48b371247b436bd2a39d2d95a21ba141c125ac7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-12T09:02:43.734 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:43 smithi090 podman[125154]: 2024-04-12 09:02:43.584339562 +0000 UTC m=+1.721047343 container cleanup 8008248b4e61ad0e130e68a9c48b371247b436bd2a39d2d95a21ba141c125ac7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=quincy-d3b5060, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-12T09:02:43.734 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:43 smithi090 bash[125154]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2 2024-04-12T09:02:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:43 smithi114 ceph-mon[89482]: pgmap v446: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:43 smithi114 ceph-mon[89482]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T09:02:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:43 smithi114 ceph-mon[89482]: osdmap e130: 8 total, 7 up, 8 in 2024-04-12T09:02:44.157 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:43 smithi090 ceph-mon[99575]: pgmap v446: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:44.157 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:43 smithi090 ceph-mon[99575]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T09:02:44.157 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:43 smithi090 ceph-mon[99575]: osdmap e130: 8 total, 7 up, 8 in 2024-04-12T09:02:44.157 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:43 smithi090 ceph-mon[95814]: pgmap v446: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:44.157 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:43 smithi090 ceph-mon[95814]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T09:02:44.157 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:43 smithi090 ceph-mon[95814]: osdmap e130: 8 total, 7 up, 8 in 2024-04-12T09:02:44.158 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:43 smithi090 podman[125172]: 2024-04-12 09:02:43.852980131 +0000 UTC m=+0.655029382 container remove 8008248b4e61ad0e130e68a9c48b371247b436bd2a39d2d95a21ba141c125ac7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, ceph=True, RELEASE=quincy-d3b5060, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:02:44.423 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:44 smithi090 podman[125290]: 2024-04-12 09:02:44.060351951 +0000 UTC m=+0.025501906 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:02:44.423 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:44 smithi090 podman[125290]: 2024-04-12 09:02:44.187698276 +0000 UTC m=+0.152848220 container create dd9ae681d9ab8419ad895921cba383c84f69d07318889ab6d852bd80a45892f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:02:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:44 smithi090 ceph-mon[95814]: osdmap e131: 8 total, 7 up, 8 in 2024-04-12T09:02:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:44 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3512816498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:44.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:44 smithi090 ceph-mon[99575]: osdmap e131: 8 total, 7 up, 8 in 2024-04-12T09:02:44.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:44 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3512816498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:44.924 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:44 smithi090 podman[125290]: 2024-04-12 09:02:44.658637509 +0000 UTC m=+0.623787455 container init dd9ae681d9ab8419ad895921cba383c84f69d07318889ab6d852bd80a45892f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True) 2024-04-12T09:02:44.924 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:44 smithi090 podman[125290]: 2024-04-12 09:02:44.661883223 +0000 UTC m=+0.627033173 container start dd9ae681d9ab8419ad895921cba383c84f69d07318889ab6d852bd80a45892f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-12T09:02:44.924 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:44 smithi090 podman[125290]: 2024-04-12 09:02:44.742090762 +0000 UTC m=+0.707240729 container attach dd9ae681d9ab8419ad895921cba383c84f69d07318889ab6d852bd80a45892f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T09:02:45.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:44 smithi114 ceph-mon[89482]: osdmap e131: 8 total, 7 up, 8 in 2024-04-12T09:02:45.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:44 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3512816498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:45.245 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:44 smithi090 podman[125290]: 2024-04-12 09:02:44.955994748 +0000 UTC m=+0.921144694 container died dd9ae681d9ab8419ad895921cba383c84f69d07318889ab6d852bd80a45892f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240409) 2024-04-12T09:02:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:45 smithi090 ceph-mon[99575]: pgmap v449: 73 pgs: 3 peering, 1 stale+active+clean, 69 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:45 smithi090 ceph-mon[99575]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-12T09:02:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:45 smithi090 ceph-mon[95814]: pgmap v449: 73 pgs: 3 peering, 1 stale+active+clean, 69 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:45 smithi090 ceph-mon[95814]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-12T09:02:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:45.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:45.924 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:45 smithi090 podman[125316]: 2024-04-12 09:02:45.539070962 +0000 UTC m=+0.578101783 container remove dd9ae681d9ab8419ad895921cba383c84f69d07318889ab6d852bd80a45892f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-12T09:02:45.924 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:45 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-12T09:02:45.924 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:45 smithi090 systemd[1]: Stopped Ceph osd.2 for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T09:02:45.924 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:45 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.2.service: Consumed 27min 18.011s CPU time. 2024-04-12T09:02:45.924 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:45 smithi090 systemd[1]: Starting Ceph osd.2 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:02:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:45 smithi114 ceph-mon[89482]: pgmap v449: 73 pgs: 3 peering, 1 stale+active+clean, 69 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-12T09:02:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:45 smithi114 ceph-mon[89482]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-12T09:02:46.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:46.118 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:02:46.262 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:45 smithi090 podman[125423]: 2024-04-12 09:02:45.89236569 +0000 UTC m=+0.020725828 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:02:46.262 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 podman[125423]: 2024-04-12 09:02:46.025807815 +0000 UTC m=+0.154167934 container create dd2646d3e259993d5e8125f9efe9b311f3cd60c30769f0e7d184e8110aaf2ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-12T09:02:46.673 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 podman[125423]: 2024-04-12 09:02:46.262509705 +0000 UTC m=+0.390869838 container init dd2646d3e259993d5e8125f9efe9b311f3cd60c30769f0e7d184e8110aaf2ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux ) 2024-04-12T09:02:46.673 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 podman[125423]: 2024-04-12 09:02:46.266016286 +0000 UTC m=+0.394376415 container start dd2646d3e259993d5e8125f9efe9b311f3cd60c30769f0e7d184e8110aaf2ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, ceph=True) 2024-04-12T09:02:46.673 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 podman[125423]: 2024-04-12 09:02:46.362380158 +0000 UTC m=+0.490740284 container attach dd2646d3e259993d5e8125f9efe9b311f3cd60c30769f0e7d184e8110aaf2ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T09:02:47.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:46 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3047577276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:47.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:46 smithi114 ceph-mon[89482]: pgmap v450: 73 pgs: 2 active+undersized+degraded, 3 peering, 6 active+undersized, 62 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-12T09:02:47.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3047577276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:47.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-mon[95814]: pgmap v450: 73 pgs: 2 active+undersized+degraded, 3 peering, 6 active+undersized, 62 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-12T09:02:47.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3047577276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:47.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-mon[99575]: pgmap v450: 73 pgs: 2 active+undersized+degraded, 3 peering, 6 active+undersized, 62 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-12T09:02:47.174 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate[125458]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T09:02:47.174 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate[125458]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T09:02:47.174 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 bash[125423]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T09:02:47.174 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 bash[125423]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T09:02:47.174 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 bash[125423]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T09:02:47.174 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 bash[125423]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T09:02:47.174 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 bash[125423]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-12T09:02:47.174 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 bash[125423]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T09:02:47.174 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 bash[125423]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-12T09:02:47.174 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate[125458]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T09:02:47.175 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate[125458]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T09:02:47.175 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate[125458]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-12T09:02:47.175 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate[125458]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T09:02:47.175 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate[125458]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-12T09:02:47.175 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:46 smithi090 podman[125423]: 2024-04-12 09:02:46.910435142 +0000 UTC m=+1.038795274 container died dd2646d3e259993d5e8125f9efe9b311f3cd60c30769f0e7d184e8110aaf2ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-b277feb, org.label-schema.license=GPLv2) 2024-04-12T09:02:47.867 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:47 smithi090 ceph-mon[95814]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T09:02:47.868 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:47 smithi090 ceph-mon[99575]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T09:02:47.868 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:47 smithi090 podman[125579]: 2024-04-12 09:02:47.526873933 +0000 UTC m=+0.611525340 container remove dd2646d3e259993d5e8125f9efe9b311f3cd60c30769f0e7d184e8110aaf2ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, ceph=True, org.label-schema.schema-version=1.0) 2024-04-12T09:02:47.868 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:47 smithi090 podman[125638]: 2024-04-12 09:02:47.641706814 +0000 UTC m=+0.018340036 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:02:47.868 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:47 smithi090 podman[125638]: 2024-04-12 09:02:47.74471777 +0000 UTC m=+0.121351020 container create d798e05b6c642be094e0d9b14796199b3d58c7ffb1987a77b2745ff4cad5567a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-12T09:02:48.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:47 smithi114 ceph-mon[89482]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T09:02:48.173 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:47 smithi090 podman[125638]: 2024-04-12 09:02:47.989423024 +0000 UTC m=+0.366056236 container init d798e05b6c642be094e0d9b14796199b3d58c7ffb1987a77b2745ff4cad5567a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-12T09:02:48.173 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:47 smithi090 podman[125638]: 2024-04-12 09:02:47.992510858 +0000 UTC m=+0.369144069 container start d798e05b6c642be094e0d9b14796199b3d58c7ffb1987a77b2745ff4cad5567a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T09:02:48.173 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:48 smithi090 ceph-osd[125656]: -- 172.21.15.90:0/855856363 <== mon.2 v2:172.21.15.90:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5650f5181680 con 0x5650f514b400 2024-04-12T09:02:48.173 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:48 smithi090 bash[125638]: d798e05b6c642be094e0d9b14796199b3d58c7ffb1987a77b2745ff4cad5567a 2024-04-12T09:02:48.173 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:48 smithi090 systemd[1]: Started Ceph osd.2 for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T09:02:48.922 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:48 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[125652]: 2024-04-12T09:02:48.576+0000 7f3849e067c0 -1 Falling back to public interface 2024-04-12T09:02:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:49 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:49 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:49 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:49 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3225507037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:49 smithi090 ceph-mon[99575]: pgmap v451: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:49.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:49 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:49.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:49 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:49 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:49 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3225507037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:49.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:49 smithi090 ceph-mon[95814]: pgmap v451: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:49.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:49 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:49.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:49 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:49.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:49 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:49.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:49 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3225507037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:49.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:49 smithi114 ceph-mon[89482]: pgmap v451: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:50.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:50 smithi090 ceph-mon[99575]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T09:02:50.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:50 smithi090 ceph-mon[95814]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T09:02:50.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:50 smithi114 ceph-mon[89482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T09:02:51.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:51 smithi090 ceph-mon[95814]: pgmap v452: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:51.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:51 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1879697150' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:51.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:51 smithi090 ceph-mon[99575]: pgmap v452: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:51.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:51 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1879697150' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:51.518 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:51 smithi114 ceph-mon[89482]: pgmap v452: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:51.519 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:51 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1879697150' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:53.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:52 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:53.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:52 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:53.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:52 smithi090 ceph-mon[99575]: pgmap v453: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:53.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:52 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:53.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:52 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:53.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:52 smithi090 ceph-mon[95814]: pgmap v453: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:53.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:52 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:53.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:52 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:53.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:52 smithi114 ceph-mon[89482]: pgmap v453: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:53.611 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:02:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:02:53] "GET /metrics HTTP/1.1" 200 34176 "" "Prometheus/2.43.0" 2024-04-12T09:02:54.235 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:53 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:54.235 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:53 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:54.235 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:53 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2160431340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:54.235 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:53 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:54.235 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:53 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:54.235 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:53 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2160431340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:54.236 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:54 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[125652]: 2024-04-12T09:02:54.058+0000 7f3849e067c0 -1 osd.2 129 log_to_monitors true 2024-04-12T09:02:54.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:53 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:54.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:53 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:54.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:53 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2160431340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:55.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:54 smithi114 ceph-mon[89482]: from='osd.2 [v2:172.21.15.90:6818/3445357160,v1:172.21.15.90:6819/3445357160]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T09:02:55.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:54 smithi114 ceph-mon[89482]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T09:02:55.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:54 smithi114 ceph-mon[89482]: pgmap v454: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:55.389 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[125652]: 2024-04-12T09:02:55.000+0000 7f383b25e700 -1 osd.2 129 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T09:02:55.390 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:54 smithi090 ceph-mon[95814]: from='osd.2 [v2:172.21.15.90:6818/3445357160,v1:172.21.15.90:6819/3445357160]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T09:02:55.390 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:54 smithi090 ceph-mon[95814]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T09:02:55.390 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:54 smithi090 ceph-mon[95814]: pgmap v454: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:55.390 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:54 smithi090 ceph-mon[99575]: from='osd.2 [v2:172.21.15.90:6818/3445357160,v1:172.21.15.90:6819/3445357160]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T09:02:55.390 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:54 smithi090 ceph-mon[99575]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T09:02:55.390 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:54 smithi090 ceph-mon[99575]: pgmap v454: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-12T09:02:56.258 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[95814]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T09:02:56.258 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[95814]: from='osd.2 [v2:172.21.15.90:6818/3445357160,v1:172.21.15.90:6819/3445357160]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T09:02:56.258 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[95814]: osdmap e132: 8 total, 7 up, 8 in 2024-04-12T09:02:56.258 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[95814]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T09:02:56.258 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1792527304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:56.258 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 5/72 objects degraded (6.944%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T09:02:56.258 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:56.259 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[99575]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T09:02:56.259 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[99575]: from='osd.2 [v2:172.21.15.90:6818/3445357160,v1:172.21.15.90:6819/3445357160]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T09:02:56.259 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[99575]: osdmap e132: 8 total, 7 up, 8 in 2024-04-12T09:02:56.259 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[99575]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T09:02:56.259 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1792527304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:56.259 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 5/72 objects degraded (6.944%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T09:02:56.259 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:55 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:55 smithi114 ceph-mon[89482]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T09:02:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:55 smithi114 ceph-mon[89482]: from='osd.2 [v2:172.21.15.90:6818/3445357160,v1:172.21.15.90:6819/3445357160]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T09:02:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:55 smithi114 ceph-mon[89482]: osdmap e132: 8 total, 7 up, 8 in 2024-04-12T09:02:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:55 smithi114 ceph-mon[89482]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T09:02:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:55 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1792527304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:55 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 5/72 objects degraded (6.944%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T09:02:56.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:55 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:57.172 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:02:57.112+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-12T09:02:57.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[99575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T09:02:57.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[99575]: osd.2 [v2:172.21.15.90:6818/3445357160,v1:172.21.15.90:6819/3445357160] boot 2024-04-12T09:02:57.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[99575]: osdmap e133: 8 total, 8 up, 8 in 2024-04-12T09:02:57.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T09:02:57.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[99575]: pgmap v457: 73 pgs: 3 peering, 1 active+undersized+degraded, 16 active+undersized, 53 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-12T09:02:57.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:57.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[95814]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[95814]: osd.2 [v2:172.21.15.90:6818/3445357160,v1:172.21.15.90:6819/3445357160] boot 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[95814]: osdmap e133: 8 total, 8 up, 8 in 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[95814]: pgmap v457: 73 pgs: 3 peering, 1 active+undersized+degraded, 16 active+undersized, 53 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:57.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:02:57.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:02:57.175 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:57 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:57.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:57 smithi114 ceph-mon[89482]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T09:02:57.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:57 smithi114 ceph-mon[89482]: osd.2 [v2:172.21.15.90:6818/3445357160,v1:172.21.15.90:6819/3445357160] boot 2024-04-12T09:02:57.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:57 smithi114 ceph-mon[89482]: osdmap e133: 8 total, 8 up, 8 in 2024-04-12T09:02:57.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:57 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T09:02:57.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:57 smithi114 ceph-mon[89482]: pgmap v457: 73 pgs: 3 peering, 1 active+undersized+degraded, 16 active+undersized, 53 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-12T09:02:57.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:57 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:57.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:57 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:57.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:57 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:02:57.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:57 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:02:57.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:57 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:02:58.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: pgmap v458: 73 pgs: 12 peering, 8 active+undersized, 53 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:02:58.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: pgmap v459: 73 pgs: 12 peering, 8 active+undersized, 53 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:02:58.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: osdmap e134: 8 total, 8 up, 8 in 2024-04-12T09:02:58.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/72 objects degraded (6.944%), 3 pgs degraded) 2024-04-12T09:02:58.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:58.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:58.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:58.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:58.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:02:58.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:02:58.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-12T09:02:58.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:02:58 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1161932255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: pgmap v458: 73 pgs: 12 peering, 8 active+undersized, 53 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:02:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: pgmap v459: 73 pgs: 12 peering, 8 active+undersized, 53 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:02:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: osdmap e134: 8 total, 8 up, 8 in 2024-04-12T09:02:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/72 objects degraded (6.944%), 3 pgs degraded) 2024-04-12T09:02:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:58.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1161932255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: pgmap v458: 73 pgs: 12 peering, 8 active+undersized, 53 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: pgmap v459: 73 pgs: 12 peering, 8 active+undersized, 53 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: osdmap e134: 8 total, 8 up, 8 in 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/72 objects degraded (6.944%), 3 pgs degraded) 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:58.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:02:58.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:02:58.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:02:58.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-12T09:02:58.425 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:02:58 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1161932255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:00.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:00 smithi114 ceph-mon[89482]: pgmap v461: 73 pgs: 12 peering, 6 active+undersized, 55 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:00.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:00 smithi090 ceph-mon[95814]: pgmap v461: 73 pgs: 12 peering, 6 active+undersized, 55 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:00.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:00 smithi090 ceph-mon[99575]: pgmap v461: 73 pgs: 12 peering, 6 active+undersized, 55 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:01.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:01 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/519776211' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:01.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:01 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:01.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:01 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:01.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:01 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/519776211' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:01.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:01 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:01.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:01.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:01 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/519776211' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:01.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:01 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:01.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:02 smithi114 ceph-mon[89482]: pgmap v462: 73 pgs: 12 peering, 61 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:02 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3271575862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:02 smithi090 ceph-mon[99575]: pgmap v462: 73 pgs: 12 peering, 61 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:02.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:02 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3271575862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:02 smithi090 ceph-mon[95814]: pgmap v462: 73 pgs: 12 peering, 61 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:02.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:02 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3271575862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:03.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:03:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:03:03] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2024-04-12T09:03:04.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:04 smithi114 ceph-mon[89482]: pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:04.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:04 smithi090 ceph-mon[99575]: pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:04.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:04 smithi090 ceph-mon[95814]: pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:05 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/693525055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:05.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:05 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/693525055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:05.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:05 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/693525055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:06.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:06 smithi090 ceph-mon[95814]: pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:06.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:06 smithi090 ceph-mon[99575]: pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:06 smithi114 ceph-mon[89482]: pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:07.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:07 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1759024626' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:07.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:07 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1759024626' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:07.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:07 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1759024626' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:08 smithi114 ceph-mon[89482]: pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:08.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:08 smithi090 ceph-mon[95814]: pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:08.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:08 smithi090 ceph-mon[99575]: pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:09.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:09 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/803707930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:09.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:09 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/803707930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:09.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:09 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/803707930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:10 smithi114 ceph-mon[89482]: pgmap v466: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:10.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:10 smithi090 ceph-mon[99575]: pgmap v466: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:10.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:10 smithi090 ceph-mon[95814]: pgmap v466: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:11 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:11 smithi090 ceph-mon[99575]: pgmap v467: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:12.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:11 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3791061855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:11 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:11 smithi090 ceph-mon[95814]: pgmap v467: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:11 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3791061855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:11 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:11 smithi114 ceph-mon[89482]: pgmap v467: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:12.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:11 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3791061855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:13.023 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:12 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:03:13.024 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:12 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:03:13.024 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:12 smithi090 ceph-mon[95814]: Upgrade: osd.3 is safe to restart 2024-04-12T09:03:13.024 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:12 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:03:13.024 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:12 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:03:13.024 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:12 smithi090 ceph-mon[99575]: Upgrade: osd.3 is safe to restart 2024-04-12T09:03:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:12 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:03:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:12 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T09:03:13.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:12 smithi114 ceph-mon[89482]: Upgrade: osd.3 is safe to restart 2024-04-12T09:03:13.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:03:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:03:13] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2024-04-12T09:03:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:13 smithi090 ceph-mon[99575]: pgmap v468: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:13 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:13 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T09:03:14.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:13 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:14.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:13 smithi090 ceph-mon[95814]: pgmap v468: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:14.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:13 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:14.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:13 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T09:03:14.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:13 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:14.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:13 smithi114 ceph-mon[89482]: pgmap v468: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:14.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:13 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:14.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:13 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T09:03:14.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:13 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:15.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:14 smithi090 ceph-mon[99575]: Upgrade: Updating osd.3 2024-04-12T09:03:15.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:14 smithi090 ceph-mon[99575]: Deploying daemon osd.3 on smithi090 2024-04-12T09:03:15.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:14 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2489984930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:15.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:14 smithi090 ceph-mon[95814]: Upgrade: Updating osd.3 2024-04-12T09:03:15.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:14 smithi090 ceph-mon[95814]: Deploying daemon osd.3 on smithi090 2024-04-12T09:03:15.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:14 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2489984930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:14 smithi114 ceph-mon[89482]: Upgrade: Updating osd.3 2024-04-12T09:03:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:14 smithi114 ceph-mon[89482]: Deploying daemon osd.3 on smithi090 2024-04-12T09:03:15.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:14 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2489984930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:16.074 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:15 smithi090 ceph-mon[99575]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:16.074 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:15 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:16.074 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:15 smithi090 ceph-mon[95814]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:16.075 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:15 smithi114 ceph-mon[89482]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:16.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:15 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:17.050 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:16 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/676520312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:17.050 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:16 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/676520312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:17.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:16 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/676520312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:17.423 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:17 smithi090 systemd[1]: Stopping Ceph osd.3 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:03:17.423 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:17 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3[61421]: 2024-04-12T09:03:17.294+0000 7fbb19281700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:03:17.423 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:17 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3[61421]: 2024-04-12T09:03:17.294+0000 7fbb19281700 -1 osd.3 134 *** Got signal Terminated *** 2024-04-12T09:03:17.423 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:17 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3[61421]: 2024-04-12T09:03:17.295+0000 7fbb19281700 -1 osd.3 134 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T09:03:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:17 smithi114 ceph-mon[89482]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:18.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:17 smithi114 ceph-mon[89482]: osd.3 marked itself down and dead 2024-04-12T09:03:18.422 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:17 smithi090 ceph-mon[99575]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:18.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:17 smithi090 ceph-mon[99575]: osd.3 marked itself down and dead 2024-04-12T09:03:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:17 smithi090 ceph-mon[95814]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:18.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:17 smithi090 ceph-mon[95814]: osd.3 marked itself down and dead 2024-04-12T09:03:18.719 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:18 smithi090 podman[130162]: 2024-04-12 09:03:18.432240773 +0000 UTC m=+1.241583522 container died 83a5641ba19afd6cbb9261760c489485f310994b8b4ddfe7adf62089cea5726f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-d3b5060, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-12T09:03:19.107 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:18 smithi090 ceph-mon[99575]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T09:03:19.107 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:18 smithi090 ceph-mon[99575]: osdmap e135: 8 total, 7 up, 8 in 2024-04-12T09:03:19.108 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:18 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2732863533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:19.108 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:18 smithi090 podman[130162]: 2024-04-12 09:03:18.822126671 +0000 UTC m=+1.631469412 container cleanup 83a5641ba19afd6cbb9261760c489485f310994b8b4ddfe7adf62089cea5726f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-12T09:03:19.108 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:18 smithi090 bash[130162]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3 2024-04-12T09:03:19.108 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:18 smithi090 ceph-mon[95814]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T09:03:19.108 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:18 smithi090 ceph-mon[95814]: osdmap e135: 8 total, 7 up, 8 in 2024-04-12T09:03:19.108 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:18 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2732863533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:19.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:18 smithi114 ceph-mon[89482]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T09:03:19.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:18 smithi114 ceph-mon[89482]: osdmap e135: 8 total, 7 up, 8 in 2024-04-12T09:03:19.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:18 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2732863533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:19.423 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:19 smithi090 podman[130182]: 2024-04-12 09:03:19.107493122 +0000 UTC m=+0.677277355 container remove 83a5641ba19afd6cbb9261760c489485f310994b8b4ddfe7adf62089cea5726f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True) 2024-04-12T09:03:19.771 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:19 smithi090 podman[130297]: 2024-04-12 09:03:19.346079773 +0000 UTC m=+0.017415734 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:03:19.772 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:19 smithi090 podman[130297]: 2024-04-12 09:03:19.458779296 +0000 UTC m=+0.130115242 container create 1715ee94f1b020fcb77c7d928db1cf77884b59fc0950a975ace882c342adafdb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:03:19.772 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:19 smithi090 podman[130297]: 2024-04-12 09:03:19.678753897 +0000 UTC m=+0.350089852 container init 1715ee94f1b020fcb77c7d928db1cf77884b59fc0950a975ace882c342adafdb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-12T09:03:19.772 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:19 smithi090 podman[130297]: 2024-04-12 09:03:19.682227345 +0000 UTC m=+0.353563295 container start 1715ee94f1b020fcb77c7d928db1cf77884b59fc0950a975ace882c342adafdb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, ceph=True, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:03:19.772 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:19 smithi090 podman[130297]: 2024-04-12 09:03:19.771360241 +0000 UTC m=+0.442696195 container attach 1715ee94f1b020fcb77c7d928db1cf77884b59fc0950a975ace882c342adafdb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-12T09:03:20.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:19 smithi090 ceph-mon[99575]: osdmap e136: 8 total, 7 up, 8 in 2024-04-12T09:03:20.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:19 smithi090 ceph-mon[99575]: pgmap v473: 73 pgs: 12 stale+active+clean, 61 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:20.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:19 smithi090 ceph-mon[95814]: osdmap e136: 8 total, 7 up, 8 in 2024-04-12T09:03:20.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:19 smithi090 ceph-mon[95814]: pgmap v473: 73 pgs: 12 stale+active+clean, 61 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:20.173 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:19 smithi090 podman[130297]: 2024-04-12 09:03:19.971984498 +0000 UTC m=+0.643320445 container died 1715ee94f1b020fcb77c7d928db1cf77884b59fc0950a975ace882c342adafdb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-12T09:03:20.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:19 smithi114 ceph-mon[89482]: osdmap e136: 8 total, 7 up, 8 in 2024-04-12T09:03:20.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:19 smithi114 ceph-mon[89482]: pgmap v473: 73 pgs: 12 stale+active+clean, 61 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:20.923 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:20 smithi090 podman[130323]: 2024-04-12 09:03:20.626149154 +0000 UTC m=+0.649422852 container remove 1715ee94f1b020fcb77c7d928db1cf77884b59fc0950a975ace882c342adafdb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:03:20.923 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:20 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-12T09:03:20.923 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:20 smithi090 systemd[1]: Stopped Ceph osd.3 for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T09:03:20.923 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:20 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.3.service: Consumed 28min 42.771s CPU time. 2024-04-12T09:03:20.923 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:20 smithi090 systemd[1]: Starting Ceph osd.3 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:03:21.423 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 podman[130447]: 2024-04-12 09:03:20.988120702 +0000 UTC m=+0.021721838 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:03:21.423 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 podman[130447]: 2024-04-12 09:03:21.111681697 +0000 UTC m=+0.145282808 container create 26d539e3ff546e28739e963175ffaec906b57ed2b53ce345d5d1f6161660736e (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.build-date=20240409, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-12T09:03:21.423 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 podman[130447]: 2024-04-12 09:03:21.324354158 +0000 UTC m=+0.357955274 container init 26d539e3ff546e28739e963175ffaec906b57ed2b53ce345d5d1f6161660736e (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T09:03:21.423 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 podman[130447]: 2024-04-12 09:03:21.328094766 +0000 UTC m=+0.361695887 container start 26d539e3ff546e28739e963175ffaec906b57ed2b53ce345d5d1f6161660736e (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-12T09:03:21.773 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 podman[130447]: 2024-04-12 09:03:21.440950745 +0000 UTC m=+0.474551856 container attach 26d539e3ff546e28739e963175ffaec906b57ed2b53ce345d5d1f6161660736e (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate, org.label-schema.build-date=20240409, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-12T09:03:22.026 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate[130464]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate[130464]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 bash[130447]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 bash[130447]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 bash[130447]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 bash[130447]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 bash[130447]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 bash[130447]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 bash[130447]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate[130464]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate[130464]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate[130464]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate[130464]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T09:03:22.027 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate[130464]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-12T09:03:22.028 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:21 smithi090 podman[130447]: 2024-04-12 09:03:21.798533669 +0000 UTC m=+0.832134792 container died 26d539e3ff546e28739e963175ffaec906b57ed2b53ce345d5d1f6161660736e (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, ceph=True) 2024-04-12T09:03:22.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:22 smithi114 ceph-mon[89482]: pgmap v474: 73 pgs: 8 stale+active+clean, 7 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:22.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:22 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1924003971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:22.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:22 smithi090 ceph-mon[95814]: pgmap v474: 73 pgs: 8 stale+active+clean, 7 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:22.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:22 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1924003971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:22.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:22 smithi090 ceph-mon[99575]: pgmap v474: 73 pgs: 8 stale+active+clean, 7 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T09:03:22.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:22 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1924003971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:22.871 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:22 smithi090 podman[130588]: 2024-04-12 09:03:22.530334444 +0000 UTC m=+0.726271862 container remove 26d539e3ff546e28739e963175ffaec906b57ed2b53ce345d5d1f6161660736e (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T09:03:22.871 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:22 smithi090 podman[130654]: 2024-04-12 09:03:22.646532017 +0000 UTC m=+0.018204802 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:03:22.871 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:22 smithi090 podman[130654]: 2024-04-12 09:03:22.749424939 +0000 UTC m=+0.121097714 container create d6cf9c940a8b59f07e3a0107ec16f0e210d271264d583a083b91b49d4742b3ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-12T09:03:23.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:23 smithi090 ceph-mon[99575]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T09:03:23.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:23 smithi090 ceph-mon[95814]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T09:03:23.173 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:22 smithi090 podman[130654]: 2024-04-12 09:03:22.993890277 +0000 UTC m=+0.365563052 container init d6cf9c940a8b59f07e3a0107ec16f0e210d271264d583a083b91b49d4742b3ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_BRANCH=HEAD) 2024-04-12T09:03:23.173 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:22 smithi090 podman[130654]: 2024-04-12 09:03:22.996908558 +0000 UTC m=+0.368581342 container start d6cf9c940a8b59f07e3a0107ec16f0e210d271264d583a083b91b49d4742b3ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-12T09:03:23.174 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:23 smithi090 ceph-osd[130673]: -- 172.21.15.90:0/3132630942 <== mon.2 v2:172.21.15.90:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f5931f7680 con 0x55f5931c0c00 2024-04-12T09:03:23.174 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:23 smithi090 bash[130654]: d6cf9c940a8b59f07e3a0107ec16f0e210d271264d583a083b91b49d4742b3ff 2024-04-12T09:03:23.174 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:23 smithi090 systemd[1]: Started Ceph osd.3 for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T09:03:23.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:23 smithi114 ceph-mon[89482]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T09:03:23.471 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:03:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:03:23] "GET /metrics HTTP/1.1" 200 34173 "" "Prometheus/2.43.0" 2024-04-12T09:03:23.922 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3[130669]: 2024-04-12T09:03:23.543+0000 7fb25001e7c0 -1 Falling back to public interface 2024-04-12T09:03:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:24 smithi114 ceph-mon[89482]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:24 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:24 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:24 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:24.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:24 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2396392393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:24.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:24 smithi090 ceph-mon[99575]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:24.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:24 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:24.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:24 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:24.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:24 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:24.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:24 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2396392393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:24.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:24 smithi090 ceph-mon[95814]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:24.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:24 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:24.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:24 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:24.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:24 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:24.424 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:24 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2396392393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:26 smithi090 ceph-mon[99575]: pgmap v476: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:26.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:26 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/459563139' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:26.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:26 smithi090 ceph-mon[95814]: pgmap v476: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:26.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:26 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/459563139' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:26.434 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:26 smithi114 ceph-mon[89482]: pgmap v476: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:26.434 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:26 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/459563139' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:28 smithi114 ceph-mon[89482]: pgmap v477: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:28 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:28 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:28 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:28 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:28 smithi090 ceph-mon[95814]: pgmap v477: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:28.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:28 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:28 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:28 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:28 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:28 smithi090 ceph-mon[99575]: pgmap v477: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:28.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:28 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:28 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:28 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:28.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:28 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:29.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:29 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1527747745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:29.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:29 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:29.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:29 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:29 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1527747745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:29 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:29.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:29 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:29.174 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:29 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3[130669]: 2024-04-12T09:03:29.078+0000 7fb25001e7c0 -1 osd.3 134 log_to_monitors true 2024-04-12T09:03:29.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:29 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1527747745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:29.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:29 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:29.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:29 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:30.358 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:30 smithi090 ceph-mon[95814]: pgmap v478: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:30.358 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:30 smithi090 ceph-mon[95814]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T09:03:30.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:30 smithi090 ceph-mon[99575]: pgmap v478: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:30.359 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:30 smithi090 ceph-mon[99575]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T09:03:30.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:30 smithi114 ceph-mon[89482]: pgmap v478: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:30.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:30 smithi114 ceph-mon[89482]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T09:03:30.673 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:03:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3[130669]: 2024-04-12T09:03:30.465+0000 7fb23d46e700 -1 osd.3 134 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T09:03:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:31 smithi114 ceph-mon[89482]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T09:03:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:31 smithi114 ceph-mon[89482]: osdmap e137: 8 total, 7 up, 8 in 2024-04-12T09:03:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:31 smithi114 ceph-mon[89482]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T09:03:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:31 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/547899514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:31 smithi114 ceph-mon[89482]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' 2024-04-12T09:03:31.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:31 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:31.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[95814]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T09:03:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[95814]: osdmap e137: 8 total, 7 up, 8 in 2024-04-12T09:03:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[95814]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T09:03:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/547899514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[95814]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' 2024-04-12T09:03:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[99575]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T09:03:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[99575]: osdmap e137: 8 total, 7 up, 8 in 2024-04-12T09:03:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[99575]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-12T09:03:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/547899514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[99575]: from='osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129]' entity='osd.3' 2024-04-12T09:03:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:31.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:32 smithi114 ceph-mon[89482]: pgmap v480: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:32 smithi114 ceph-mon[89482]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T09:03:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:32 smithi114 ceph-mon[89482]: osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129] boot 2024-04-12T09:03:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:32 smithi114 ceph-mon[89482]: osdmap e138: 8 total, 8 up, 8 in 2024-04-12T09:03:32.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:32 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T09:03:32.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:03:32.219+0000 7efe43a91700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-12T09:03:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-mon[95814]: pgmap v480: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-mon[95814]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T09:03:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-mon[95814]: osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129] boot 2024-04-12T09:03:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-mon[95814]: osdmap e138: 8 total, 8 up, 8 in 2024-04-12T09:03:32.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T09:03:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-mon[99575]: pgmap v480: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-mon[99575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T09:03:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-mon[99575]: osd.3 [v2:172.21.15.90:6826/4110199129,v1:172.21.15.90:6827/4110199129] boot 2024-04-12T09:03:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-mon[99575]: osdmap e138: 8 total, 8 up, 8 in 2024-04-12T09:03:32.424 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:32 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T09:03:32.728 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 4 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: pgmap v482: 73 pgs: 8 peering, 3 active+undersized+degraded, 18 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: pgmap v483: 73 pgs: 8 peering, 3 active+undersized+degraded, 18 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:33.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: osdmap e139: 8 total, 8 up, 8 in 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2784183482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: pgmap v482: 73 pgs: 8 peering, 3 active+undersized+degraded, 18 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: pgmap v483: 73 pgs: 8 peering, 3 active+undersized+degraded, 18 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:33.282 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:33.283 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:33.283 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:33.283 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:33.283 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:33.283 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T09:03:33.283 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T09:03:33.283 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-12T09:03:33.283 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: osdmap e139: 8 total, 8 up, 8 in 2024-04-12T09:03:33.283 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2784183482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:33.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:33.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:33.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:33.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:03:33.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: pgmap v482: 73 pgs: 8 peering, 3 active+undersized+degraded, 18 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:33.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: pgmap v483: 73 pgs: 8 peering, 3 active+undersized+degraded, 18 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:33.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:33.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:33.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:33.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:33.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:33.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T09:03:33.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T09:03:33.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-12T09:03:33.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: osdmap e139: 8 total, 8 up, 8 in 2024-04-12T09:03:33.369 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:33 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2784183482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T09:03:33.492 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade pause' 2024-04-12T09:03:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:03:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:03:33] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-12T09:03:33.752 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:35.406 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:35 smithi090 ceph-mon[99575]: pgmap v485: 73 pgs: 8 peering, 3 active+undersized+degraded, 18 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:35.406 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:35 smithi090 ceph-mon[95814]: pgmap v485: 73 pgs: 8 peering, 3 active+undersized+degraded, 18 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:35 smithi114 ceph-mon[89482]: pgmap v485: 73 pgs: 8 peering, 3 active+undersized+degraded, 18 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-12T09:03:35.634 INFO:teuthology.orchestra.run.smithi090.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:03:36.430 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch ps' 2024-04-12T09:03:36.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:36 smithi090 ceph-mon[99575]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T09:03:36.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:36 smithi090 ceph-mon[99575]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:03:36.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:36 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:36.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:36 smithi090 ceph-mon[95814]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T09:03:36.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:36 smithi090 ceph-mon[95814]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:03:36.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:36 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:36.689 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:36 smithi114 ceph-mon[89482]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T09:03:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:36 smithi114 ceph-mon[89482]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:03:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:36 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:37.833 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:37 smithi090 ceph-mon[99575]: pgmap v486: 73 pgs: 8 peering, 1 active+undersized+degraded, 5 active+undersized, 59 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-12T09:03:37.833 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:37 smithi090 ceph-mon[99575]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-12T09:03:37.833 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:37 smithi090 ceph-mon[95814]: pgmap v486: 73 pgs: 8 peering, 1 active+undersized+degraded, 5 active+undersized, 59 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-12T09:03:37.833 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:37 smithi090 ceph-mon[95814]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-12T09:03:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:37 smithi114 ceph-mon[89482]: pgmap v486: 73 pgs: 8 peering, 1 active+undersized+degraded, 5 active+undersized, 59 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-12T09:03:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:37 smithi114 ceph-mon[89482]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-12T09:03:38.634 INFO:teuthology.orchestra.run.smithi090.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T09:03:38.634 INFO:teuthology.orchestra.run.smithi090.stdout:alertmanager.a smithi090 *:9093,9094 running (15m) 11s ago 21m 24.2M - 0.25.0 c8568f914cd2 d11ba2dc752a 2024-04-12T09:03:38.634 INFO:teuthology.orchestra.run.smithi090.stdout:grafana.a smithi114 *:3000 error 11s ago 21m - - 2024-04-12T09:03:38.634 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi090.rxdrqj smithi090 running (20m) 11s ago 20m 22.9M - 17.2.7-827-gd3b5060a 21de9b7369ad f627bce7c272 2024-04-12T09:03:38.634 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi114.nbzjob smithi114 running (20m) 11s ago 20m 16.7M - 17.2.7-827-gd3b5060a 21de9b7369ad 60cb65499580 2024-04-12T09:03:38.634 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.x smithi114 *:8443,9283,8765 running (14m) 11s ago 25m 455M - 19.0.0-2053-gb277feb2 668895944f0e 60c0ca408f91 2024-04-12T09:03:38.634 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.y smithi090 *:8443,9283,8765 running (15m) 11s ago 26m 513M - 19.0.0-2053-gb277feb2 668895944f0e b61a0f582c61 2024-04-12T09:03:38.634 INFO:teuthology.orchestra.run.smithi090.stdout:mon.a smithi090 running (13m) 11s ago 27m 66.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e ad8458aaf579 2024-04-12T09:03:38.634 INFO:teuthology.orchestra.run.smithi090.stdout:mon.b smithi114 running (10m) 11s ago 25m 51.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e cd2de45bf1bf 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:mon.c smithi090 running (12m) 11s ago 25m 56.3M 2048M 19.0.0-2053-gb277feb2 668895944f0e 6bfdbaae0edc 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.a smithi090 *:9100 running (15m) 11s ago 21m 17.1M - 1.5.0 0da6a335fe13 687747c6880c 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.b smithi114 *:9100 running (15m) 11s ago 21m 15.6M - 1.5.0 0da6a335fe13 e89762981b55 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:osd.0 smithi090 running (8m) 11s ago 24m 614M 2480M 19.0.0-2053-gb277feb2 668895944f0e c245fffdfa56 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:osd.1 smithi090 running (4m) 11s ago 24m 338M 2480M 19.0.0-2053-gb277feb2 668895944f0e be4538ab3be1 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:osd.2 smithi090 running (50s) 11s ago 24m 41.5M 2480M 19.0.0-2053-gb277feb2 668895944f0e d798e05b6c64 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:osd.3 smithi090 running (15s) 11s ago 23m 15.0M 2480M 19.0.0-2053-gb277feb2 668895944f0e d6cf9c940a8b 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:osd.4 smithi114 running (23m) 11s ago 23m 542M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 15d0d8563392 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:osd.5 smithi114 running (22m) 11s ago 22m 1935M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 54049e134882 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:osd.6 smithi114 running (22m) 11s ago 22m 573M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 86dfedc8629b 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:osd.7 smithi114 running (22m) 11s ago 22m 566M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad dda597b19712 2024-04-12T09:03:38.635 INFO:teuthology.orchestra.run.smithi090.stdout:prometheus.a smithi114 *:9095 running (14m) 11s ago 21m 61.4M - 2.43.0 a07b618ecd1d f009118ca428 2024-04-12T09:03:38.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:38 smithi090 ceph-mon[99575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-12T09:03:38.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:38 smithi090 ceph-mon[95814]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-12T09:03:39.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:38 smithi114 ceph-mon[89482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-12T09:03:39.330 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch ps' 2024-04-12T09:03:39.587 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:39.945 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:39 smithi090 ceph-mon[99575]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:39.945 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:39 smithi090 ceph-mon[99575]: from='client.45161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T09:03:39.945 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:39 smithi090 ceph-mon[95814]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:39.945 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:39 smithi090 ceph-mon[95814]: from='client.45161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T09:03:40.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:39 smithi114 ceph-mon[89482]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:40.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:39 smithi114 ceph-mon[89482]: from='client.45161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:alertmanager.a smithi090 *:9093,9094 running (15m) 14s ago 21m 24.2M - 0.25.0 c8568f914cd2 d11ba2dc752a 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:grafana.a smithi114 *:3000 error 14s ago 21m - - 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi090.rxdrqj smithi090 running (20m) 14s ago 20m 22.9M - 17.2.7-827-gd3b5060a 21de9b7369ad f627bce7c272 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:mds.foo.smithi114.nbzjob smithi114 running (20m) 14s ago 20m 16.7M - 17.2.7-827-gd3b5060a 21de9b7369ad 60cb65499580 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.x smithi114 *:8443,9283,8765 running (14m) 14s ago 25m 455M - 19.0.0-2053-gb277feb2 668895944f0e 60c0ca408f91 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.y smithi090 *:8443,9283,8765 running (15m) 14s ago 26m 513M - 19.0.0-2053-gb277feb2 668895944f0e b61a0f582c61 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:mon.a smithi090 running (13m) 14s ago 27m 66.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e ad8458aaf579 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:mon.b smithi114 running (10m) 14s ago 25m 51.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e cd2de45bf1bf 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:mon.c smithi090 running (12m) 14s ago 25m 56.3M 2048M 19.0.0-2053-gb277feb2 668895944f0e 6bfdbaae0edc 2024-04-12T09:03:41.497 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.a smithi090 *:9100 running (15m) 14s ago 21m 17.1M - 1.5.0 0da6a335fe13 687747c6880c 2024-04-12T09:03:41.498 INFO:teuthology.orchestra.run.smithi090.stdout:node-exporter.b smithi114 *:9100 running (15m) 14s ago 21m 15.6M - 1.5.0 0da6a335fe13 e89762981b55 2024-04-12T09:03:41.498 INFO:teuthology.orchestra.run.smithi090.stdout:osd.0 smithi090 running (8m) 14s ago 24m 614M 2480M 19.0.0-2053-gb277feb2 668895944f0e c245fffdfa56 2024-04-12T09:03:41.498 INFO:teuthology.orchestra.run.smithi090.stdout:osd.1 smithi090 running (5m) 14s ago 24m 338M 2480M 19.0.0-2053-gb277feb2 668895944f0e be4538ab3be1 2024-04-12T09:03:41.498 INFO:teuthology.orchestra.run.smithi090.stdout:osd.2 smithi090 running (53s) 14s ago 24m 41.5M 2480M 19.0.0-2053-gb277feb2 668895944f0e d798e05b6c64 2024-04-12T09:03:41.498 INFO:teuthology.orchestra.run.smithi090.stdout:osd.3 smithi090 running (18s) 14s ago 23m 15.0M 2480M 19.0.0-2053-gb277feb2 668895944f0e d6cf9c940a8b 2024-04-12T09:03:41.498 INFO:teuthology.orchestra.run.smithi090.stdout:osd.4 smithi114 running (23m) 14s ago 23m 542M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 15d0d8563392 2024-04-12T09:03:41.498 INFO:teuthology.orchestra.run.smithi090.stdout:osd.5 smithi114 running (23m) 14s ago 23m 1935M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 54049e134882 2024-04-12T09:03:41.498 INFO:teuthology.orchestra.run.smithi090.stdout:osd.6 smithi114 running (22m) 14s ago 22m 573M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 86dfedc8629b 2024-04-12T09:03:41.498 INFO:teuthology.orchestra.run.smithi090.stdout:osd.7 smithi114 running (22m) 14s ago 22m 566M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad dda597b19712 2024-04-12T09:03:41.498 INFO:teuthology.orchestra.run.smithi090.stdout:prometheus.a smithi114 *:9095 running (14m) 14s ago 21m 61.4M - 2.43.0 a07b618ecd1d f009118ca428 2024-04-12T09:03:42.092 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:41 smithi090 ceph-mon[95814]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:42.093 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:41 smithi090 ceph-mon[99575]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:42.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:41 smithi114 ceph-mon[89482]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:42.119 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph versions' 2024-04-12T09:03:42.373 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:42.839 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:42 smithi090 ceph-mon[95814]: from='client.45167 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T09:03:42.839 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:42 smithi090 ceph-mon[99575]: from='client.45167 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T09:03:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:42 smithi114 ceph-mon[89482]: from='client.45167 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T09:03:43.494 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:03:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:03:43] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-12T09:03:43.826 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:43 smithi090 ceph-mon[99575]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:43.826 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:43 smithi090 ceph-mon[95814]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:43 smithi114 ceph-mon[89482]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout: "mon": { 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 3 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout: "mgr": { 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout: "osd": { 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)": 4, 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 4 2024-04-12T09:03:44.356 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-12T09:03:44.357 INFO:teuthology.orchestra.run.smithi090.stdout: "mds": { 2024-04-12T09:03:44.357 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)": 2 2024-04-12T09:03:44.357 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-12T09:03:44.357 INFO:teuthology.orchestra.run.smithi090.stdout: "overall": { 2024-04-12T09:03:44.357 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)": 6, 2024-04-12T09:03:44.357 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 9 2024-04-12T09:03:44.357 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-12T09:03:44.357 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-12T09:03:44.997 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:44 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2697612030' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:44.998 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:44 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2697612030' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:45.022 DEBUG:teuthology.parallel:result is None 2024-04-12T09:03:45.023 INFO:teuthology.run_tasks:Running task print... 2024-04-12T09:03:45.033 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-12T09:03:45.033 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-12T09:03:45.056 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-12T09:03:45.056 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-12T09:03:45.056 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph mgr dump --format=json 2024-04-12T09:03:45.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:44 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2697612030' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T09:03:45.310 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:45 smithi114 ceph-mon[89482]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:45 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:46.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:45 smithi090 ceph-mon[99575]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:46.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:45 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:46.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:45 smithi090 ceph-mon[95814]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:45 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:46.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:03:47.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:46 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:47.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:46 smithi090 ceph-mon[95814]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:47.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:46 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:47.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:46 smithi090 ceph-mon[99575]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:47.243 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:03:47.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:46 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:47.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:46 smithi114 ceph-mon[89482]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:47.953 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":39,"flags":0,"active_gid":34104,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6800","nonce":1467705464},{"type":"v1","addr":"172.21.15.90:6801","nonce":1467705464}]},"active_addr":"172.21.15.90:6801/1467705464","active_change":"2024-04-12T08:50:00.306292+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25186,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.90:8443/","prometheus":"http://172.21.15.90:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":83,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":2353467386}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":2568604397}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":3651681151}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":1923515862}]}]} 2024-04-12T09:03:47.961 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-12T09:03:47.961 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-12T09:03:47.962 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T09:03:48.215 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:48.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:48.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:48.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:03:48.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.337 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1589186689' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:03:48.338 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1589186689' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T09:03:48.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:48.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:48.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:48.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:03:48.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.339 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:47 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:03:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1589186689' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T09:03:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:48.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:03:48.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:03:48.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:03:48.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:48.368 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:47 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:03:49.998 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:03:49.998 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":139,"fsid":"96735376-f8a7-11ee-bc8c-c7b262605968","created":"2024-04-12T08:36:35.370883+0000","modified":"2024-04-12T09:03:32.463350+0000","last_up_change":"2024-04-12T09:03:31.461639+0000","last_in_change":"2024-04-12T08:41:16.832300+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T08:39:38.877058+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T08:42:17.012082+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T08:43:26.507787+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T08:43:27.356729+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"3e888cb8-b0cb-4cb8-95ae-7dd29d090968","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":88,"up_from":95,"up_thru":138,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":3578880330},{"type":"v1","addr":"172.21.15.90:6803","nonce":3578880330}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":3578880330},{"type":"v1","addr":"172.21.15.90:6805","nonce":3578880330}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":3578880330},{"type":"v1","addr":"172.21.15.90:6809","nonce":3578880330}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":3578880330},{"type":"v1","addr":"172.21.15.90:6807","nonce":3578880330}]},"public_addr":"172.21.15.90:6803/3578880330","cluster_addr":"172.21.15.90:6805/3578880330","heartbeat_back_addr":"172.21.15.90:6809/3578880330","heartbeat_front_addr":"172.21.15.90:6807/3578880330","state":["exists","up"]},{"osd":1,"uuid":"637213e4-7019-4430-ab01-b7196988f82c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":112,"up_from":116,"up_thru":138,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":1619969530},{"type":"v1","addr":"172.21.15.90:6811","nonce":1619969530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":1619969530},{"type":"v1","addr":"172.21.15.90:6813","nonce":1619969530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":1619969530},{"type":"v1","addr":"172.21.15.90:6817","nonce":1619969530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":1619969530},{"type":"v1","addr":"172.21.15.90:6815","nonce":1619969530}]},"public_addr":"172.21.15.90:6811/1619969530","cluster_addr":"172.21.15.90:6813/1619969530","heartbeat_back_addr":"172.21.15.90:6817/1619969530","heartbeat_front_addr":"172.21.15.90:6815/1619969530","state":["exists","up"]},{"osd":2,"uuid":"99513c85-6468-4377-85b8-3dc52f0d0f2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":129,"up_from":133,"up_thru":138,"down_at":130,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":3445357160},{"type":"v1","addr":"172.21.15.90:6819","nonce":3445357160}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":3445357160},{"type":"v1","addr":"172.21.15.90:6821","nonce":3445357160}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":3445357160},{"type":"v1","addr":"172.21.15.90:6825","nonce":3445357160}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":3445357160},{"type":"v1","addr":"172.21.15.90:6823","nonce":3445357160}]},"public_addr":"172.21.15.90:6819/3445357160","cluster_addr":"172.21.15.90:6821/3445357160","heartbeat_back_addr":"172.21.15.90:6825/3445357160","heartbeat_front_addr":"172.21.15.90:6823/3445357160","state":["exists","up"]},{"osd":3,"uuid":"afd01784-8987-4954-9be5-57d950beb611","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":134,"up_from":138,"up_thru":138,"down_at":135,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":4110199129},{"type":"v1","addr":"172.21.15.90:6827","nonce":4110199129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":4110199129},{"type":"v1","addr":"172.21.15.90:6829","nonce":4110199129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":4110199129},{"type":"v1","addr":"172.21.15.90:6833","nonce":4110199129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":4110199129},{"type":"v1","addr":"172.21.15.90:6831","nonce":4110199129}]},"public_addr":"172.21.15.90:6827/4110199129","cluster_addr":"172.21.15.90:6829/4110199129","heartbeat_back_addr":"172.21.15.90:6833/4110199129","heartbeat_front_addr":"172.21.15.90:6831/4110199129","state":["exists","up"]},{"osd":4,"uuid":"596f9e87-d99e-423c-902f-f14c5dcdcf36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":138,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6801","nonce":1802592095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6803","nonce":1802592095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6807","nonce":1802592095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6805","nonce":1802592095}]},"public_addr":"172.21.15.114:6801/1802592095","cluster_addr":"172.21.15.114:6803/1802592095","heartbeat_back_addr":"172.21.15.114:6807/1802592095","heartbeat_front_addr":"172.21.15.114:6805/1802592095","state":["exists","up"]},{"osd":5,"uuid":"b2374027-89a9-4670-9593-3ddf9393b286","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":138,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6809","nonce":117630080}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6811","nonce":117630080}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6815","nonce":117630080}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6813","nonce":117630080}]},"public_addr":"172.21.15.114:6809/117630080","cluster_addr":"172.21.15.114:6811/117630080","heartbeat_back_addr":"172.21.15.114:6815/117630080","heartbeat_front_addr":"172.21.15.114:6813/117630080","state":["exists","up"]},{"osd":6,"uuid":"487cdd03-1123-4386-9237-97b2bc6dd92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":135,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6817","nonce":2139728372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6819","nonce":2139728372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6823","nonce":2139728372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6821","nonce":2139728372}]},"public_addr":"172.21.15.114:6817/2139728372","cluster_addr":"172.21.15.114:6819/2139728372","heartbeat_back_addr":"172.21.15.114:6823/2139728372","heartbeat_front_addr":"172.21.15.114:6821/2139728372","state":["exists","up"]},{"osd":7,"uuid":"0d46acbb-c271-4f1f-b5ba-090a44443308","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":135,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6825","nonce":3563377268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6827","nonce":3563377268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6831","nonce":3563377268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6829","nonce":3563377268}]},"public_addr":"172.21.15.114:6825/3563377268","cluster_addr":"172.21.15.114:6827/3563377268","heartbeat_back_addr":"172.21.15.114:6831/3563377268","heartbeat_front_addr":"172.21.15.114:6829/3563377268","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:38:46.298090+0000","dead_epoch":88},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:11.083177+0000","dead_epoch":112},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:35.186704+0000","dead_epoch":129},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:59.618196+0000","dead_epoch":134},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:21.910327+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:44.541214+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:08.064023+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:31.949563+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.6","mappings":[{"from":4,"to":2}]},{"pgid":"3.a","mappings":[{"from":4,"to":0}]},{"pgid":"3.14","mappings":[{"from":4,"to":0}]},{"pgid":"3.15","mappings":[{"from":4,"to":1},{"from":7,"to":0}]},{"pgid":"3.19","mappings":[{"from":3,"to":0}]},{"pgid":"3.1c","mappings":[{"from":4,"to":0}]},{"pgid":"4.f","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.19","mappings":[{"from":3,"to":5}]},{"pgid":"4.1c","mappings":[{"from":1,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/4226855639":"2024-04-13T08:50:00.305565+0000","172.21.15.90:0/1455732421":"2024-04-13T08:50:00.305565+0000","172.21.15.90:0/3631859869":"2024-04-13T08:37:02.805251+0000","172.21.15.114:0/44322435":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/794297450":"2024-04-13T08:47:20.876475+0000","172.21.15.90:6801/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3912205664":"2024-04-13T08:47:20.876475+0000","172.21.15.90:0/3166816614":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6800/3500986148":"2024-04-13T08:47:20.876475+0000","172.21.15.90:0/1103578375":"2024-04-13T08:50:00.305565+0000","172.21.15.90:0/2050499985":"2024-04-13T08:50:00.305565+0000","172.21.15.90:0/380935950":"2024-04-13T08:50:00.305565+0000","172.21.15.90:6800/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6801/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6800/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/1763471199":"2024-04-13T08:47:20.876475+0000","172.21.15.114:6835/4185706983":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/806687008":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6801/3500986148":"2024-04-13T08:47:20.876475+0000","172.21.15.114:0/570127456":"2024-04-13T08:48:46.636934+0000","172.21.15.114:0/2435762721":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/3964952044":"2024-04-13T08:42:02.589107+0000","172.21.15.114:0/1070611450":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/4275803179":"2024-04-13T08:47:20.876475+0000","172.21.15.90:6800/769831032":"2024-04-13T08:50:00.305565+0000","172.21.15.90:0/721288352":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6800/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/769831032":"2024-04-13T08:50:00.305565+0000","172.21.15.114:0/654160568":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/1996176058":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/577805764":"2024-04-13T08:42:02.589107+0000","172.21.15.114:0/3782240454":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/3357701831":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/3408352967":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.114:6834/4185706983":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/3110959800":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/71273579":"2024-04-13T08:50:00.305565+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T09:03:50.642 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:50 smithi090 ceph-mon[99575]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:50.642 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:50 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2305058646' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T09:03:50.643 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:50 smithi090 ceph-mon[95814]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:50.643 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:50 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2305058646' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T09:03:50.669 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-12T09:03:50.669 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T09:03:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:50 smithi114 ceph-mon[89482]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:50 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2305058646' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T09:03:50.926 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:52.558 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:52 smithi090 ceph-mon[99575]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:52.559 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:52 smithi090 ceph-mon[95814]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:52.754 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:03:52.754 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":139,"fsid":"96735376-f8a7-11ee-bc8c-c7b262605968","created":"2024-04-12T08:36:35.370883+0000","modified":"2024-04-12T09:03:32.463350+0000","last_up_change":"2024-04-12T09:03:31.461639+0000","last_in_change":"2024-04-12T08:41:16.832300+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T08:39:38.877058+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T08:42:17.012082+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T08:43:26.507787+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T08:43:27.356729+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"3e888cb8-b0cb-4cb8-95ae-7dd29d090968","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":88,"up_from":95,"up_thru":138,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":3578880330},{"type":"v1","addr":"172.21.15.90:6803","nonce":3578880330}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":3578880330},{"type":"v1","addr":"172.21.15.90:6805","nonce":3578880330}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":3578880330},{"type":"v1","addr":"172.21.15.90:6809","nonce":3578880330}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":3578880330},{"type":"v1","addr":"172.21.15.90:6807","nonce":3578880330}]},"public_addr":"172.21.15.90:6803/3578880330","cluster_addr":"172.21.15.90:6805/3578880330","heartbeat_back_addr":"172.21.15.90:6809/3578880330","heartbeat_front_addr":"172.21.15.90:6807/3578880330","state":["exists","up"]},{"osd":1,"uuid":"637213e4-7019-4430-ab01-b7196988f82c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":112,"up_from":116,"up_thru":138,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":1619969530},{"type":"v1","addr":"172.21.15.90:6811","nonce":1619969530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":1619969530},{"type":"v1","addr":"172.21.15.90:6813","nonce":1619969530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":1619969530},{"type":"v1","addr":"172.21.15.90:6817","nonce":1619969530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":1619969530},{"type":"v1","addr":"172.21.15.90:6815","nonce":1619969530}]},"public_addr":"172.21.15.90:6811/1619969530","cluster_addr":"172.21.15.90:6813/1619969530","heartbeat_back_addr":"172.21.15.90:6817/1619969530","heartbeat_front_addr":"172.21.15.90:6815/1619969530","state":["exists","up"]},{"osd":2,"uuid":"99513c85-6468-4377-85b8-3dc52f0d0f2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":129,"up_from":133,"up_thru":138,"down_at":130,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":3445357160},{"type":"v1","addr":"172.21.15.90:6819","nonce":3445357160}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":3445357160},{"type":"v1","addr":"172.21.15.90:6821","nonce":3445357160}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":3445357160},{"type":"v1","addr":"172.21.15.90:6825","nonce":3445357160}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":3445357160},{"type":"v1","addr":"172.21.15.90:6823","nonce":3445357160}]},"public_addr":"172.21.15.90:6819/3445357160","cluster_addr":"172.21.15.90:6821/3445357160","heartbeat_back_addr":"172.21.15.90:6825/3445357160","heartbeat_front_addr":"172.21.15.90:6823/3445357160","state":["exists","up"]},{"osd":3,"uuid":"afd01784-8987-4954-9be5-57d950beb611","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":134,"up_from":138,"up_thru":138,"down_at":135,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":4110199129},{"type":"v1","addr":"172.21.15.90:6827","nonce":4110199129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":4110199129},{"type":"v1","addr":"172.21.15.90:6829","nonce":4110199129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":4110199129},{"type":"v1","addr":"172.21.15.90:6833","nonce":4110199129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":4110199129},{"type":"v1","addr":"172.21.15.90:6831","nonce":4110199129}]},"public_addr":"172.21.15.90:6827/4110199129","cluster_addr":"172.21.15.90:6829/4110199129","heartbeat_back_addr":"172.21.15.90:6833/4110199129","heartbeat_front_addr":"172.21.15.90:6831/4110199129","state":["exists","up"]},{"osd":4,"uuid":"596f9e87-d99e-423c-902f-f14c5dcdcf36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":138,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6801","nonce":1802592095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6803","nonce":1802592095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6807","nonce":1802592095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":1802592095},{"type":"v1","addr":"172.21.15.114:6805","nonce":1802592095}]},"public_addr":"172.21.15.114:6801/1802592095","cluster_addr":"172.21.15.114:6803/1802592095","heartbeat_back_addr":"172.21.15.114:6807/1802592095","heartbeat_front_addr":"172.21.15.114:6805/1802592095","state":["exists","up"]},{"osd":5,"uuid":"b2374027-89a9-4670-9593-3ddf9393b286","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":138,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6809","nonce":117630080}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6811","nonce":117630080}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6815","nonce":117630080}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":117630080},{"type":"v1","addr":"172.21.15.114:6813","nonce":117630080}]},"public_addr":"172.21.15.114:6809/117630080","cluster_addr":"172.21.15.114:6811/117630080","heartbeat_back_addr":"172.21.15.114:6815/117630080","heartbeat_front_addr":"172.21.15.114:6813/117630080","state":["exists","up"]},{"osd":6,"uuid":"487cdd03-1123-4386-9237-97b2bc6dd92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":135,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6817","nonce":2139728372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6819","nonce":2139728372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6823","nonce":2139728372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":2139728372},{"type":"v1","addr":"172.21.15.114:6821","nonce":2139728372}]},"public_addr":"172.21.15.114:6817/2139728372","cluster_addr":"172.21.15.114:6819/2139728372","heartbeat_back_addr":"172.21.15.114:6823/2139728372","heartbeat_front_addr":"172.21.15.114:6821/2139728372","state":["exists","up"]},{"osd":7,"uuid":"0d46acbb-c271-4f1f-b5ba-090a44443308","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":135,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6825","nonce":3563377268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6827","nonce":3563377268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6831","nonce":3563377268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":3563377268},{"type":"v1","addr":"172.21.15.114:6829","nonce":3563377268}]},"public_addr":"172.21.15.114:6825/3563377268","cluster_addr":"172.21.15.114:6827/3563377268","heartbeat_back_addr":"172.21.15.114:6831/3563377268","heartbeat_front_addr":"172.21.15.114:6829/3563377268","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:38:46.298090+0000","dead_epoch":88},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:11.083177+0000","dead_epoch":112},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:35.186704+0000","dead_epoch":129},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:39:59.618196+0000","dead_epoch":134},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:21.910327+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:40:44.541214+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:08.064023+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T08:41:31.949563+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.6","mappings":[{"from":4,"to":2}]},{"pgid":"3.a","mappings":[{"from":4,"to":0}]},{"pgid":"3.14","mappings":[{"from":4,"to":0}]},{"pgid":"3.15","mappings":[{"from":4,"to":1},{"from":7,"to":0}]},{"pgid":"3.19","mappings":[{"from":3,"to":0}]},{"pgid":"3.1c","mappings":[{"from":4,"to":0}]},{"pgid":"4.f","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.19","mappings":[{"from":3,"to":5}]},{"pgid":"4.1c","mappings":[{"from":1,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/4226855639":"2024-04-13T08:50:00.305565+0000","172.21.15.90:0/1455732421":"2024-04-13T08:50:00.305565+0000","172.21.15.90:0/3631859869":"2024-04-13T08:37:02.805251+0000","172.21.15.114:0/44322435":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/794297450":"2024-04-13T08:47:20.876475+0000","172.21.15.90:6801/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/3912205664":"2024-04-13T08:47:20.876475+0000","172.21.15.90:0/3166816614":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6800/3500986148":"2024-04-13T08:47:20.876475+0000","172.21.15.90:0/1103578375":"2024-04-13T08:50:00.305565+0000","172.21.15.90:0/2050499985":"2024-04-13T08:50:00.305565+0000","172.21.15.90:0/380935950":"2024-04-13T08:50:00.305565+0000","172.21.15.90:6800/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6801/1335936558":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6800/841793822":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/1763471199":"2024-04-13T08:47:20.876475+0000","172.21.15.114:6835/4185706983":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/806687008":"2024-04-13T08:37:26.567479+0000","172.21.15.90:6801/3500986148":"2024-04-13T08:47:20.876475+0000","172.21.15.114:0/570127456":"2024-04-13T08:48:46.636934+0000","172.21.15.114:0/2435762721":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/3964952044":"2024-04-13T08:42:02.589107+0000","172.21.15.114:0/1070611450":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/4275803179":"2024-04-13T08:47:20.876475+0000","172.21.15.90:6800/769831032":"2024-04-13T08:50:00.305565+0000","172.21.15.90:0/721288352":"2024-04-13T08:37:02.805251+0000","172.21.15.90:6800/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/769831032":"2024-04-13T08:50:00.305565+0000","172.21.15.114:0/654160568":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/1996176058":"2024-04-13T08:37:26.567479+0000","172.21.15.90:0/577805764":"2024-04-13T08:42:02.589107+0000","172.21.15.114:0/3782240454":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/3357701831":"2024-04-13T08:42:02.589107+0000","172.21.15.90:0/3408352967":"2024-04-13T08:42:02.589107+0000","172.21.15.90:6801/3620748501":"2024-04-13T08:42:02.589107+0000","172.21.15.114:6834/4185706983":"2024-04-13T08:48:46.636934+0000","172.21.15.90:0/3110959800":"2024-04-13T08:37:02.805251+0000","172.21.15.90:0/71273579":"2024-04-13T08:50:00.305565+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T09:03:52.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:52 smithi114 ceph-mon[89482]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:53.510 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-12T09:03:53.510 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-12T09:03:53.511 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-12T09:03:53.511 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-12T09:03:53.511 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-12T09:03:53.512 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-12T09:03:53.512 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-12T09:03:53.513 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-12T09:03:53.551 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:53 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2147645212' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T09:03:53.551 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:03:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:03:53] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.43.0" 2024-04-12T09:03:53.552 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:53 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2147645212' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T09:03:53.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:53 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2147645212' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T09:03:53.970 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:53.981 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:53.991 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:54.017 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:54.018 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:54.024 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:54.028 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:54.030 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:03:54.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:54 smithi090 ceph-mon[99575]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:54.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:54 smithi090 ceph-mon[95814]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:54 smithi114 ceph-mon[89482]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:56.729 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:56 smithi090 ceph-mon[99575]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:56.729 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:56 smithi090 ceph-mon[95814]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:56 smithi114 ceph-mon[89482]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:58.855 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:03:58 smithi090 ceph-mon[99575]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:58.855 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:03:58 smithi090 ceph-mon[95814]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:03:58 smithi114 ceph-mon[89482]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:03:59.568 INFO:teuthology.orchestra.run.smithi090.stdout:571230650382 2024-04-12T09:03:59.569 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-12T09:04:00.821 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:00 smithi090 ceph-mon[99575]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:00.821 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:00 smithi090 ceph-mon[95814]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:00 smithi114 ceph-mon[89482]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:01.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:01 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:01.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:01.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:02.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:02 smithi090 ceph-mon[99575]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:02.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:02 smithi090 ceph-mon[95814]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:02.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:02 smithi114 ceph-mon[89482]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:03.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:04:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:04:03] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.43.0" 2024-04-12T09:04:04.660 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:04 smithi090 ceph-mon[99575]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:04.661 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:04 smithi090 ceph-mon[95814]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:04.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:04 smithi114 ceph-mon[89482]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:06.455 INFO:teuthology.orchestra.run.smithi090.stdout:592705486856 2024-04-12T09:04:06.455 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-12T09:04:06.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:06 smithi090 ceph-mon[95814]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:06.813 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:06 smithi090 ceph-mon[99575]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:06 smithi114 ceph-mon[89482]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:07.463 INFO:teuthology.orchestra.run.smithi090.stdout:201863463188 2024-04-12T09:04:07.463 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-12T09:04:07.799 INFO:teuthology.orchestra.run.smithi090.stdout:133143986462 2024-04-12T09:04:07.800 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-12T09:04:08.197 INFO:teuthology.orchestra.run.smithi090.stdout:167503724825 2024-04-12T09:04:08.198 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-12T09:04:08.765 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:08 smithi090 ceph-mon[95814]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:08.766 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:08 smithi090 ceph-mon[99575]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:08 smithi114 ceph-mon[89482]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:09.074 INFO:teuthology.orchestra.run.smithi090.stdout:408021893228 2024-04-12T09:04:09.075 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-12T09:04:10.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:10 smithi090 ceph-mon[99575]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:10.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:10 smithi090 ceph-mon[95814]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:10.721 INFO:teuthology.orchestra.run.smithi090.stdout:498216206400 2024-04-12T09:04:10.722 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-12T09:04:10.725 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:10.764 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:10.764 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:10.765 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:10.768 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:10.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:10 smithi114 ceph-mon[89482]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:12.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:12 smithi090 ceph-mon[95814]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:12.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:12 smithi090 ceph-mon[99575]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:12 smithi114 ceph-mon[89482]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:13.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:04:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:04:13] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.43.0" 2024-04-12T09:04:14.768 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:14 smithi090 ceph-mon[99575]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:14.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:14 smithi090 ceph-mon[95814]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:14.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:14 smithi114 ceph-mon[89482]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:15.562 INFO:teuthology.orchestra.run.smithi090.stdout:201863463190 2024-04-12T09:04:16.589 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:16 smithi090 ceph-mon[99575]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:16.589 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:16 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:16.589 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:16 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/865347200' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T09:04:16.589 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:16 smithi090 ceph-mon[95814]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:16.589 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:16.589 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:16 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/865347200' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T09:04:16.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:16 smithi114 ceph-mon[89482]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:16.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:16 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:16.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:16 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/865347200' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T09:04:17.506 INFO:teuthology.orchestra.run.smithi090.stdout:167503724828 2024-04-12T09:04:17.725 INFO:teuthology.orchestra.run.smithi090.stdout:408021893230 2024-04-12T09:04:18.205 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724825 got 167503724828 for osd.5 2024-04-12T09:04:18.205 DEBUG:teuthology.parallel:result is None 2024-04-12T09:04:18.657 INFO:teuthology.orchestra.run.smithi090.stdout:592705486859 2024-04-12T09:04:18.844 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:18 smithi090 ceph-mon[99575]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:18.844 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:18 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4119630155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T09:04:18.844 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:18 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3559153071' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T09:04:18.845 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:18 smithi090 ceph-mon[95814]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:18.845 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:18 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4119630155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T09:04:18.845 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:18 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3559153071' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T09:04:18.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:18 smithi114 ceph-mon[89482]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:18.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:18 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4119630155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T09:04:18.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:18 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3559153071' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T09:04:19.305 INFO:tasks.cephadm.ceph_manager.ceph:need seq 408021893228 got 408021893230 for osd.0 2024-04-12T09:04:19.306 DEBUG:teuthology.parallel:result is None 2024-04-12T09:04:19.589 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:19.600 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:19 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2037557980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T09:04:19.600 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:19 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2037557980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T09:04:19.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:19 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2037557980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T09:04:20.250 INFO:tasks.cephadm.ceph_manager.ceph:need seq 592705486856 got 592705486859 for osd.3 2024-04-12T09:04:20.250 DEBUG:teuthology.parallel:result is None 2024-04-12T09:04:20.827 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:20 smithi090 ceph-mon[99575]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:20.827 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:20 smithi090 ceph-mon[95814]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:20 smithi114 ceph-mon[89482]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:21.254 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:21.681 INFO:teuthology.orchestra.run.smithi090.stdout:236223201555 2024-04-12T09:04:21.682 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-12T09:04:22.627 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:22 smithi090 ceph-mon[99575]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:22.628 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:22 smithi090 ceph-mon[95814]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:22.657 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463188 got 201863463190 for osd.6 2024-04-12T09:04:22.658 DEBUG:teuthology.parallel:result is None 2024-04-12T09:04:22.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:22 smithi114 ceph-mon[89482]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:23.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:04:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:04:23] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.43.0" 2024-04-12T09:04:24.579 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:24 smithi090 ceph-mon[99575]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:24.580 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:24 smithi090 ceph-mon[95814]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:24 smithi114 ceph-mon[89482]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:25.183 INFO:teuthology.orchestra.run.smithi090.stdout:571230650387 2024-04-12T09:04:25.409 INFO:teuthology.orchestra.run.smithi090.stdout:133143986466 2024-04-12T09:04:25.840 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2388369607' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T09:04:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/787621393' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T09:04:25.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2388369607' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T09:04:25.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/787621393' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T09:04:25.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2388369607' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T09:04:25.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/787621393' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T09:04:26.271 INFO:tasks.cephadm.ceph_manager.ceph:need seq 571230650382 got 571230650387 for osd.2 2024-04-12T09:04:26.271 DEBUG:teuthology.parallel:result is None 2024-04-12T09:04:26.712 INFO:teuthology.orchestra.run.smithi090.stdout:498216206404 2024-04-12T09:04:26.762 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:26 smithi090 ceph-mon[99575]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:26.762 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:26 smithi090 ceph-mon[95814]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:26 smithi114 ceph-mon[89482]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:27.785 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:27 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/418768945' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T09:04:27.786 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:27 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/418768945' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T09:04:27.814 INFO:tasks.cephadm.ceph_manager.ceph:need seq 498216206400 got 498216206404 for osd.1 2024-04-12T09:04:27.815 DEBUG:teuthology.parallel:result is None 2024-04-12T09:04:27.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:27 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/418768945' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T09:04:28.648 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986462 got 133143986466 for osd.4 2024-04-12T09:04:28.648 DEBUG:teuthology.parallel:result is None 2024-04-12T09:04:28.795 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:28 smithi090 ceph-mon[99575]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:28.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:28 smithi090 ceph-mon[95814]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:28 smithi114 ceph-mon[89482]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:29.521 INFO:teuthology.orchestra.run.smithi090.stdout:236223201557 2024-04-12T09:04:30.231 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201555 got 236223201557 for osd.7 2024-04-12T09:04:30.231 DEBUG:teuthology.parallel:result is None 2024-04-12T09:04:30.231 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-12T09:04:30.231 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T09:04:30.490 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:30.850 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:30 smithi090 ceph-mon[99575]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:30.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:30 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2917264286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T09:04:30.851 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:30 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:30.851 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:30 smithi090 ceph-mon[95814]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:30.851 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:30 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2917264286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T09:04:30.851 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:30 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:30.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:30 smithi114 ceph-mon[89482]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:30.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:30 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2917264286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T09:04:30.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:30 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:32.340 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:04:32.342 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-12T09:04:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:32 smithi090 ceph-mon[99575]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:32.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:32 smithi090 ceph-mon[95814]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:32 smithi114 ceph-mon[89482]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:33.069 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":517,"stamp":"2024-04-12T09:04:31.391795+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":1775938,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2697048,"kb_used_data":15312,"kb_used_omap":29,"kb_used_meta":2681570,"kb_avail":747264168,"statfs":{"total":767960285184,"available":765198508032,"internally_reserved":0,"allocated":15679488,"data_stored":9035667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30174,"internal_metadata":2745928226},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002617"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":117,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.466882+0000","last_change":"2024-04-12T09:02:57.000425+0000","last_active":"2024-04-12T09:03:31.466882+0000","last_peered":"2024-04-12T09:03:31.466882+0000","last_clean":"2024-04-12T09:03:31.466882+0000","last_became_active":"2024-04-12T09:02:56.999444+0000","last_became_peered":"2024-04-12T09:02:56.999444+0000","last_unstale":"2024-04-12T09:03:31.466882+0000","last_undegraded":"2024-04-12T09:03:31.466882+0000","last_fullsized":"2024-04-12T09:03:31.466882+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:43:56.538903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":89,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479711+0000","last_change":"2024-04-12T09:03:32.479711+0000","last_active":"2024-04-12T09:03:32.479711+0000","last_peered":"2024-04-12T09:03:32.479711+0000","last_clean":"2024-04-12T09:03:32.479711+0000","last_became_active":"2024-04-12T09:03:32.478500+0000","last_became_peered":"2024-04-12T09:03:32.478500+0000","last_unstale":"2024-04-12T09:03:32.479711+0000","last_undegraded":"2024-04-12T09:03:32.479711+0000","last_fullsized":"2024-04-12T09:03:32.479711+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:12:34.581151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":122,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473038+0000","last_change":"2024-04-12T08:58:56.743931+0000","last_active":"2024-04-12T09:03:32.473038+0000","last_peered":"2024-04-12T09:03:32.473038+0000","last_clean":"2024-04-12T09:03:32.473038+0000","last_became_active":"2024-04-12T08:58:56.740909+0000","last_became_peered":"2024-04-12T08:58:56.740909+0000","last_unstale":"2024-04-12T09:03:32.473038+0000","last_undegraded":"2024-04-12T09:03:32.473038+0000","last_fullsized":"2024-04-12T09:03:32.473038+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:54:05.956455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":121,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.486300+0000","last_change":"2024-04-12T09:03:32.486300+0000","last_active":"2024-04-12T09:03:32.486300+0000","last_peered":"2024-04-12T09:03:32.486300+0000","last_clean":"2024-04-12T09:03:32.486300+0000","last_became_active":"2024-04-12T09:03:32.485911+0000","last_became_peered":"2024-04-12T09:03:32.485911+0000","last_unstale":"2024-04-12T09:03:32.486300+0000","last_undegraded":"2024-04-12T09:03:32.486300+0000","last_fullsized":"2024-04-12T09:03:32.486300+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:02:35.257943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":107,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.483915+0000","last_change":"2024-04-12T09:03:32.483915+0000","last_active":"2024-04-12T09:03:32.483915+0000","last_peered":"2024-04-12T09:03:32.483915+0000","last_clean":"2024-04-12T09:03:32.483915+0000","last_became_active":"2024-04-12T09:03:32.483345+0000","last_became_peered":"2024-04-12T09:03:32.483345+0000","last_unstale":"2024-04-12T09:03:32.483915+0000","last_undegraded":"2024-04-12T09:03:32.483915+0000","last_fullsized":"2024-04-12T09:03:32.483915+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:42:17.502077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":97,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.471313+0000","last_change":"2024-04-12T08:58:56.774042+0000","last_active":"2024-04-12T09:03:31.471313+0000","last_peered":"2024-04-12T09:03:31.471313+0000","last_clean":"2024-04-12T09:03:31.471313+0000","last_became_active":"2024-04-12T08:58:56.773622+0000","last_became_peered":"2024-04-12T08:58:56.773622+0000","last_unstale":"2024-04-12T09:03:31.471313+0000","last_undegraded":"2024-04-12T09:03:31.471313+0000","last_fullsized":"2024-04-12T09:03:31.471313+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:08:11.813135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479868+0000","last_change":"2024-04-12T09:03:32.479868+0000","last_active":"2024-04-12T09:03:32.479868+0000","last_peered":"2024-04-12T09:03:32.479868+0000","last_clean":"2024-04-12T09:03:32.479868+0000","last_became_active":"2024-04-12T09:03:32.479576+0000","last_became_peered":"2024-04-12T09:03:32.479576+0000","last_unstale":"2024-04-12T09:03:32.479868+0000","last_undegraded":"2024-04-12T09:03:32.479868+0000","last_fullsized":"2024-04-12T09:03:32.479868+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:28:47.455110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":89,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479067+0000","last_change":"2024-04-12T09:03:32.479067+0000","last_active":"2024-04-12T09:03:32.479067+0000","last_peered":"2024-04-12T09:03:32.479067+0000","last_clean":"2024-04-12T09:03:32.479067+0000","last_became_active":"2024-04-12T09:03:32.478746+0000","last_became_peered":"2024-04-12T09:03:32.478746+0000","last_unstale":"2024-04-12T09:03:32.479067+0000","last_undegraded":"2024-04-12T09:03:32.479067+0000","last_fullsized":"2024-04-12T09:03:32.479067+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:40:04.214931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469850+0000","last_change":"2024-04-12T08:58:56.736460+0000","last_active":"2024-04-12T09:03:31.469850+0000","last_peered":"2024-04-12T09:03:31.469850+0000","last_clean":"2024-04-12T09:03:31.469850+0000","last_became_active":"2024-04-12T08:58:56.736126+0000","last_became_peered":"2024-04-12T08:58:56.736126+0000","last_unstale":"2024-04-12T09:03:31.469850+0000","last_undegraded":"2024-04-12T09:03:31.469850+0000","last_fullsized":"2024-04-12T09:03:31.469850+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:29:35.656864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.482168+0000","last_change":"2024-04-12T09:03:32.482168+0000","last_active":"2024-04-12T09:03:32.482168+0000","last_peered":"2024-04-12T09:03:32.482168+0000","last_clean":"2024-04-12T09:03:32.482168+0000","last_became_active":"2024-04-12T09:03:32.481615+0000","last_became_peered":"2024-04-12T09:03:32.481615+0000","last_unstale":"2024-04-12T09:03:32.482168+0000","last_undegraded":"2024-04-12T09:03:32.482168+0000","last_fullsized":"2024-04-12T09:03:32.482168+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:52:49.384832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":121,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.486344+0000","last_change":"2024-04-12T09:03:32.486344+0000","last_active":"2024-04-12T09:03:32.486344+0000","last_peered":"2024-04-12T09:03:32.486344+0000","last_clean":"2024-04-12T09:03:32.486344+0000","last_became_active":"2024-04-12T09:03:32.485987+0000","last_became_peered":"2024-04-12T09:03:32.485987+0000","last_unstale":"2024-04-12T09:03:32.486344+0000","last_undegraded":"2024-04-12T09:03:32.486344+0000","last_fullsized":"2024-04-12T09:03:32.486344+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:44:10.209457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":119,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473223+0000","last_change":"2024-04-12T09:02:56.999939+0000","last_active":"2024-04-12T09:03:32.473223+0000","last_peered":"2024-04-12T09:03:32.473223+0000","last_clean":"2024-04-12T09:03:32.473223+0000","last_became_active":"2024-04-12T09:02:56.999624+0000","last_became_peered":"2024-04-12T09:02:56.999624+0000","last_unstale":"2024-04-12T09:03:32.473223+0000","last_undegraded":"2024-04-12T09:03:32.473223+0000","last_fullsized":"2024-04-12T09:03:32.473223+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:40:02.678154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":107,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.477074+0000","last_change":"2024-04-12T08:55:18.487199+0000","last_active":"2024-04-12T09:03:32.477074+0000","last_peered":"2024-04-12T09:03:32.477074+0000","last_clean":"2024-04-12T09:03:32.477074+0000","last_became_active":"2024-04-12T08:55:18.391671+0000","last_became_peered":"2024-04-12T08:55:18.391671+0000","last_unstale":"2024-04-12T09:03:32.477074+0000","last_undegraded":"2024-04-12T09:03:32.477074+0000","last_fullsized":"2024-04-12T09:03:32.477074+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:41:36.634168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":98,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.467663+0000","last_change":"2024-04-12T08:58:56.736878+0000","last_active":"2024-04-12T09:03:31.467663+0000","last_peered":"2024-04-12T09:03:31.467663+0000","last_clean":"2024-04-12T09:03:31.467663+0000","last_became_active":"2024-04-12T08:58:56.736472+0000","last_became_peered":"2024-04-12T08:58:56.736472+0000","last_unstale":"2024-04-12T09:03:31.467663+0000","last_undegraded":"2024-04-12T09:03:31.467663+0000","last_fullsized":"2024-04-12T09:03:31.467663+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:48:57.668138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":100,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.474796+0000","last_change":"2024-04-12T09:02:56.999551+0000","last_active":"2024-04-12T09:03:31.474796+0000","last_peered":"2024-04-12T09:03:31.474796+0000","last_clean":"2024-04-12T09:03:31.474796+0000","last_became_active":"2024-04-12T09:02:56.999117+0000","last_became_peered":"2024-04-12T09:02:56.999117+0000","last_unstale":"2024-04-12T09:03:31.474796+0000","last_undegraded":"2024-04-12T09:03:31.474796+0000","last_fullsized":"2024-04-12T09:03:31.474796+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:20:49.562177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.482027+0000","last_change":"2024-04-12T09:03:32.482027+0000","last_active":"2024-04-12T09:03:32.482027+0000","last_peered":"2024-04-12T09:03:32.482027+0000","last_clean":"2024-04-12T09:03:32.482027+0000","last_became_active":"2024-04-12T09:03:32.481606+0000","last_became_peered":"2024-04-12T09:03:32.481606+0000","last_unstale":"2024-04-12T09:03:32.482027+0000","last_undegraded":"2024-04-12T09:03:32.482027+0000","last_fullsized":"2024-04-12T09:03:32.482027+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:20:17.182218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":119,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.485440+0000","last_change":"2024-04-12T09:03:32.485440+0000","last_active":"2024-04-12T09:03:32.485440+0000","last_peered":"2024-04-12T09:03:32.485440+0000","last_clean":"2024-04-12T09:03:32.485440+0000","last_became_active":"2024-04-12T09:03:32.485197+0000","last_became_peered":"2024-04-12T09:03:32.485197+0000","last_unstale":"2024-04-12T09:03:32.485440+0000","last_undegraded":"2024-04-12T09:03:32.485440+0000","last_fullsized":"2024-04-12T09:03:32.485440+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:16:10.437274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479277+0000","last_change":"2024-04-12T09:03:32.479277+0000","last_active":"2024-04-12T09:03:32.479277+0000","last_peered":"2024-04-12T09:03:32.479277+0000","last_clean":"2024-04-12T09:03:32.479277+0000","last_became_active":"2024-04-12T09:03:32.478957+0000","last_became_peered":"2024-04-12T09:03:32.478957+0000","last_unstale":"2024-04-12T09:03:32.479277+0000","last_undegraded":"2024-04-12T09:03:32.479277+0000","last_fullsized":"2024-04-12T09:03:32.479277+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:01:32.853192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.473836+0000","last_change":"2024-04-12T08:43:28.133594+0000","last_active":"2024-04-12T09:03:31.473836+0000","last_peered":"2024-04-12T09:03:31.473836+0000","last_clean":"2024-04-12T09:03:31.473836+0000","last_became_active":"2024-04-12T08:43:28.133305+0000","last_became_peered":"2024-04-12T08:43:28.133305+0000","last_unstale":"2024-04-12T09:03:31.473836+0000","last_undegraded":"2024-04-12T09:03:31.473836+0000","last_fullsized":"2024-04-12T09:03:31.473836+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:09:26.576479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":120,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.486054+0000","last_change":"2024-04-12T09:03:32.486054+0000","last_active":"2024-04-12T09:03:32.486054+0000","last_peered":"2024-04-12T09:03:32.486054+0000","last_clean":"2024-04-12T09:03:32.486054+0000","last_became_active":"2024-04-12T09:03:32.485729+0000","last_became_peered":"2024-04-12T09:03:32.485729+0000","last_unstale":"2024-04-12T09:03:32.486054+0000","last_undegraded":"2024-04-12T09:03:32.486054+0000","last_fullsized":"2024-04-12T09:03:32.486054+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:07:13.286240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":106,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.471382+0000","last_change":"2024-04-12T08:55:17.353074+0000","last_active":"2024-04-12T09:03:31.471382+0000","last_peered":"2024-04-12T09:03:31.471382+0000","last_clean":"2024-04-12T09:03:31.471382+0000","last_became_active":"2024-04-12T08:55:17.352780+0000","last_became_peered":"2024-04-12T08:55:17.352780+0000","last_unstale":"2024-04-12T09:03:31.471382+0000","last_undegraded":"2024-04-12T09:03:31.471382+0000","last_fullsized":"2024-04-12T09:03:31.471382+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:29:03.405163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.481571+0000","last_change":"2024-04-12T09:03:32.481571+0000","last_active":"2024-04-12T09:03:32.481571+0000","last_peered":"2024-04-12T09:03:32.481571+0000","last_clean":"2024-04-12T09:03:32.481571+0000","last_became_active":"2024-04-12T09:03:32.481207+0000","last_became_peered":"2024-04-12T09:03:32.481207+0000","last_unstale":"2024-04-12T09:03:32.481571+0000","last_undegraded":"2024-04-12T09:03:32.481571+0000","last_fullsized":"2024-04-12T09:03:32.481571+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:30:16.040167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":99,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.475837+0000","last_change":"2024-04-12T08:55:17.426384+0000","last_active":"2024-04-12T09:03:31.475837+0000","last_peered":"2024-04-12T09:03:31.475837+0000","last_clean":"2024-04-12T09:03:31.475837+0000","last_became_active":"2024-04-12T08:55:17.425999+0000","last_became_peered":"2024-04-12T08:55:17.425999+0000","last_unstale":"2024-04-12T09:03:31.475837+0000","last_undegraded":"2024-04-12T09:03:31.475837+0000","last_fullsized":"2024-04-12T09:03:31.475837+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:10:35.822055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469441+0000","last_change":"2024-04-12T08:43:28.866806+0000","last_active":"2024-04-12T09:03:31.469441+0000","last_peered":"2024-04-12T09:03:31.469441+0000","last_clean":"2024-04-12T09:03:31.469441+0000","last_became_active":"2024-04-12T08:43:28.866472+0000","last_became_peered":"2024-04-12T08:43:28.866472+0000","last_unstale":"2024-04-12T09:03:31.469441+0000","last_undegraded":"2024-04-12T09:03:31.469441+0000","last_fullsized":"2024-04-12T09:03:31.469441+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:23.104376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":100,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.475388+0000","last_change":"2024-04-12T08:58:56.737605+0000","last_active":"2024-04-12T09:03:31.475388+0000","last_peered":"2024-04-12T09:03:31.475388+0000","last_clean":"2024-04-12T09:03:31.475388+0000","last_became_active":"2024-04-12T08:58:56.736799+0000","last_became_peered":"2024-04-12T08:58:56.736799+0000","last_unstale":"2024-04-12T09:03:31.475388+0000","last_undegraded":"2024-04-12T09:03:31.475388+0000","last_fullsized":"2024-04-12T09:03:31.475388+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:45.117695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.480130+0000","last_change":"2024-04-12T09:03:32.480130+0000","last_active":"2024-04-12T09:03:32.480130+0000","last_peered":"2024-04-12T09:03:32.480130+0000","last_clean":"2024-04-12T09:03:32.480130+0000","last_became_active":"2024-04-12T09:03:32.479831+0000","last_became_peered":"2024-04-12T09:03:32.479831+0000","last_unstale":"2024-04-12T09:03:32.480130+0000","last_undegraded":"2024-04-12T09:03:32.480130+0000","last_fullsized":"2024-04-12T09:03:32.480130+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:50:04.535521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":82,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.474915+0000","last_change":"2024-04-12T08:43:28.133358+0000","last_active":"2024-04-12T09:03:31.474915+0000","last_peered":"2024-04-12T09:03:31.474915+0000","last_clean":"2024-04-12T09:03:31.474915+0000","last_became_active":"2024-04-12T08:43:28.133112+0000","last_became_peered":"2024-04-12T08:43:28.133112+0000","last_unstale":"2024-04-12T09:03:31.474915+0000","last_undegraded":"2024-04-12T09:03:31.474915+0000","last_fullsized":"2024-04-12T09:03:31.474915+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:29:38.251048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":97,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.470947+0000","last_change":"2024-04-12T08:58:56.773685+0000","last_active":"2024-04-12T09:03:31.470947+0000","last_peered":"2024-04-12T09:03:31.470947+0000","last_clean":"2024-04-12T09:03:31.470947+0000","last_became_active":"2024-04-12T08:58:56.773314+0000","last_became_peered":"2024-04-12T08:58:56.773314+0000","last_unstale":"2024-04-12T09:03:31.470947+0000","last_undegraded":"2024-04-12T09:03:31.470947+0000","last_fullsized":"2024-04-12T09:03:31.470947+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:29:42.013427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.477270+0000","last_change":"2024-04-12T09:03:32.477270+0000","last_active":"2024-04-12T09:03:32.477270+0000","last_peered":"2024-04-12T09:03:32.477270+0000","last_clean":"2024-04-12T09:03:32.477270+0000","last_became_active":"2024-04-12T09:03:32.476901+0000","last_became_peered":"2024-04-12T09:03:32.476901+0000","last_unstale":"2024-04-12T09:03:32.477270+0000","last_undegraded":"2024-04-12T09:03:32.477270+0000","last_fullsized":"2024-04-12T09:03:32.477270+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:38:19.824709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":121,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.476741+0000","last_change":"2024-04-12T08:58:56.744712+0000","last_active":"2024-04-12T09:03:32.476741+0000","last_peered":"2024-04-12T09:03:32.476741+0000","last_clean":"2024-04-12T09:03:32.476741+0000","last_became_active":"2024-04-12T08:58:56.744396+0000","last_became_peered":"2024-04-12T08:58:56.744396+0000","last_unstale":"2024-04-12T09:03:32.476741+0000","last_undegraded":"2024-04-12T09:03:32.476741+0000","last_fullsized":"2024-04-12T09:03:32.476741+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:14:40.787811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":71,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.484437+0000","last_change":"2024-04-12T09:03:32.484437+0000","last_active":"2024-04-12T09:03:32.484437+0000","last_peered":"2024-04-12T09:03:32.484437+0000","last_clean":"2024-04-12T09:03:32.484437+0000","last_became_active":"2024-04-12T09:03:32.483444+0000","last_became_peered":"2024-04-12T09:03:32.483444+0000","last_unstale":"2024-04-12T09:03:32.484437+0000","last_undegraded":"2024-04-12T09:03:32.484437+0000","last_fullsized":"2024-04-12T09:03:32.484437+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:19.582083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.484024+0000","last_change":"2024-04-12T09:03:32.484024+0000","last_active":"2024-04-12T09:03:32.484024+0000","last_peered":"2024-04-12T09:03:32.484024+0000","last_clean":"2024-04-12T09:03:32.484024+0000","last_became_active":"2024-04-12T09:03:32.483693+0000","last_became_peered":"2024-04-12T09:03:32.483693+0000","last_unstale":"2024-04-12T09:03:32.484024+0000","last_undegraded":"2024-04-12T09:03:32.484024+0000","last_fullsized":"2024-04-12T09:03:32.484024+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:45:51.457544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":122,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.470249+0000","last_change":"2024-04-12T08:58:56.774315+0000","last_active":"2024-04-12T09:03:31.470249+0000","last_peered":"2024-04-12T09:03:31.470249+0000","last_clean":"2024-04-12T09:03:31.470249+0000","last_became_active":"2024-04-12T08:58:56.774000+0000","last_became_peered":"2024-04-12T08:58:56.774000+0000","last_unstale":"2024-04-12T09:03:31.470249+0000","last_undegraded":"2024-04-12T09:03:31.470249+0000","last_fullsized":"2024-04-12T09:03:31.470249+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:19:25.327127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.467100+0000","last_change":"2024-04-12T09:02:56.999564+0000","last_active":"2024-04-12T09:03:31.467100+0000","last_peered":"2024-04-12T09:03:31.467100+0000","last_clean":"2024-04-12T09:03:31.467100+0000","last_became_active":"2024-04-12T09:02:56.999215+0000","last_became_peered":"2024-04-12T09:02:56.999215+0000","last_unstale":"2024-04-12T09:03:31.467100+0000","last_undegraded":"2024-04-12T09:03:31.467100+0000","last_fullsized":"2024-04-12T09:03:31.467100+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:51:28.380649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.466872+0000","last_change":"2024-04-12T09:02:56.999590+0000","last_active":"2024-04-12T09:03:31.466872+0000","last_peered":"2024-04-12T09:03:31.466872+0000","last_clean":"2024-04-12T09:03:31.466872+0000","last_became_active":"2024-04-12T09:02:56.999119+0000","last_became_peered":"2024-04-12T09:02:56.999119+0000","last_unstale":"2024-04-12T09:03:31.466872+0000","last_undegraded":"2024-04-12T09:03:31.466872+0000","last_fullsized":"2024-04-12T09:03:31.466872+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:44:45.124730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.467149+0000","last_change":"2024-04-12T08:58:56.738328+0000","last_active":"2024-04-12T09:03:31.467149+0000","last_peered":"2024-04-12T09:03:31.467149+0000","last_clean":"2024-04-12T09:03:31.467149+0000","last_became_active":"2024-04-12T08:58:56.737689+0000","last_became_peered":"2024-04-12T08:58:56.737689+0000","last_unstale":"2024-04-12T09:03:31.467149+0000","last_undegraded":"2024-04-12T09:03:31.467149+0000","last_fullsized":"2024-04-12T09:03:31.467149+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:27:04.300888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.482407+0000","last_change":"2024-04-12T09:03:32.482407+0000","last_active":"2024-04-12T09:03:32.482407+0000","last_peered":"2024-04-12T09:03:32.482407+0000","last_clean":"2024-04-12T09:03:32.482407+0000","last_became_active":"2024-04-12T09:03:32.481981+0000","last_became_peered":"2024-04-12T09:03:32.481981+0000","last_unstale":"2024-04-12T09:03:32.482407+0000","last_undegraded":"2024-04-12T09:03:32.482407+0000","last_fullsized":"2024-04-12T09:03:32.482407+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:28:43.592976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":95,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.478444+0000","last_change":"2024-04-12T09:03:32.478444+0000","last_active":"2024-04-12T09:03:32.478444+0000","last_peered":"2024-04-12T09:03:32.478444+0000","last_clean":"2024-04-12T09:03:32.478444+0000","last_became_active":"2024-04-12T09:03:32.478110+0000","last_became_peered":"2024-04-12T09:03:32.478110+0000","last_unstale":"2024-04-12T09:03:32.478444+0000","last_undegraded":"2024-04-12T09:03:32.478444+0000","last_fullsized":"2024-04-12T09:03:32.478444+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:57:39.815554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":118,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473714+0000","last_change":"2024-04-12T08:58:56.742998+0000","last_active":"2024-04-12T09:03:32.473714+0000","last_peered":"2024-04-12T09:03:32.473714+0000","last_clean":"2024-04-12T09:03:32.473714+0000","last_became_active":"2024-04-12T08:58:56.742632+0000","last_became_peered":"2024-04-12T08:58:56.742632+0000","last_unstale":"2024-04-12T09:03:32.473714+0000","last_undegraded":"2024-04-12T09:03:32.473714+0000","last_fullsized":"2024-04-12T09:03:32.473714+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:27:20.286312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.477229+0000","last_change":"2024-04-12T09:03:32.477229+0000","last_active":"2024-04-12T09:03:32.477229+0000","last_peered":"2024-04-12T09:03:32.477229+0000","last_clean":"2024-04-12T09:03:32.477229+0000","last_became_active":"2024-04-12T09:03:32.476902+0000","last_became_peered":"2024-04-12T09:03:32.476902+0000","last_unstale":"2024-04-12T09:03:32.477229+0000","last_undegraded":"2024-04-12T09:03:32.477229+0000","last_fullsized":"2024-04-12T09:03:32.477229+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:29:32.950731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":110,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473276+0000","last_change":"2024-04-12T08:58:56.739554+0000","last_active":"2024-04-12T09:03:32.473276+0000","last_peered":"2024-04-12T09:03:32.473276+0000","last_clean":"2024-04-12T09:03:32.473276+0000","last_became_active":"2024-04-12T08:58:56.737612+0000","last_became_peered":"2024-04-12T08:58:56.737612+0000","last_unstale":"2024-04-12T09:03:32.473276+0000","last_undegraded":"2024-04-12T09:03:32.473276+0000","last_fullsized":"2024-04-12T09:03:32.473276+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:30.076059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":103,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469512+0000","last_change":"2024-04-12T08:58:56.775286+0000","last_active":"2024-04-12T09:03:31.469512+0000","last_peered":"2024-04-12T09:03:31.469512+0000","last_clean":"2024-04-12T09:03:31.469512+0000","last_became_active":"2024-04-12T08:58:56.774900+0000","last_became_peered":"2024-04-12T08:58:56.774900+0000","last_unstale":"2024-04-12T09:03:31.469512+0000","last_undegraded":"2024-04-12T09:03:31.469512+0000","last_fullsized":"2024-04-12T09:03:31.469512+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:02:51.435526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.467547+0000","last_change":"2024-04-12T08:55:17.354270+0000","last_active":"2024-04-12T09:03:31.467547+0000","last_peered":"2024-04-12T09:03:31.467547+0000","last_clean":"2024-04-12T09:03:31.467547+0000","last_became_active":"2024-04-12T08:55:17.353940+0000","last_became_peered":"2024-04-12T08:55:17.353940+0000","last_unstale":"2024-04-12T09:03:31.467547+0000","last_undegraded":"2024-04-12T09:03:31.467547+0000","last_fullsized":"2024-04-12T09:03:31.467547+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:44.298234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":128,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.475801+0000","last_change":"2024-04-12T09:03:32.475801+0000","last_active":"2024-04-12T09:03:32.475801+0000","last_peered":"2024-04-12T09:03:32.475801+0000","last_clean":"2024-04-12T09:03:32.475801+0000","last_became_active":"2024-04-12T09:03:32.475490+0000","last_became_peered":"2024-04-12T09:03:32.475490+0000","last_unstale":"2024-04-12T09:03:32.475801+0000","last_undegraded":"2024-04-12T09:03:32.475801+0000","last_fullsized":"2024-04-12T09:03:32.475801+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:07:08.146234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":92,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479411+0000","last_change":"2024-04-12T09:03:32.479411+0000","last_active":"2024-04-12T09:03:32.479411+0000","last_peered":"2024-04-12T09:03:32.479411+0000","last_clean":"2024-04-12T09:03:32.479411+0000","last_became_active":"2024-04-12T09:03:32.478995+0000","last_became_peered":"2024-04-12T09:03:32.478995+0000","last_unstale":"2024-04-12T09:03:32.479411+0000","last_undegraded":"2024-04-12T09:03:32.479411+0000","last_fullsized":"2024-04-12T09:03:32.479411+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:14.135569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":118,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473165+0000","last_change":"2024-04-12T09:02:56.999377+0000","last_active":"2024-04-12T09:03:32.473165+0000","last_peered":"2024-04-12T09:03:32.473165+0000","last_clean":"2024-04-12T09:03:32.473165+0000","last_became_active":"2024-04-12T09:02:56.999004+0000","last_became_peered":"2024-04-12T09:02:56.999004+0000","last_unstale":"2024-04-12T09:03:32.473165+0000","last_undegraded":"2024-04-12T09:03:32.473165+0000","last_fullsized":"2024-04-12T09:03:32.473165+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:43:41.903567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":113,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473778+0000","last_change":"2024-04-12T08:58:56.772398+0000","last_active":"2024-04-12T09:03:32.473778+0000","last_peered":"2024-04-12T09:03:32.473778+0000","last_clean":"2024-04-12T09:03:32.473778+0000","last_became_active":"2024-04-12T08:58:56.771926+0000","last_became_peered":"2024-04-12T08:58:56.771926+0000","last_unstale":"2024-04-12T09:03:32.473778+0000","last_undegraded":"2024-04-12T09:03:32.473778+0000","last_fullsized":"2024-04-12T09:03:32.473778+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:59:29.002877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":103,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469493+0000","last_change":"2024-04-12T09:02:57.000385+0000","last_active":"2024-04-12T09:03:31.469493+0000","last_peered":"2024-04-12T09:03:31.469493+0000","last_clean":"2024-04-12T09:03:31.469493+0000","last_became_active":"2024-04-12T09:02:57.000052+0000","last_became_peered":"2024-04-12T09:02:57.000052+0000","last_unstale":"2024-04-12T09:03:31.469493+0000","last_undegraded":"2024-04-12T09:03:31.469493+0000","last_fullsized":"2024-04-12T09:03:31.469493+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:58:03.857415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"83'161","reported_seq":478,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.474613+0000","last_change":"2024-04-12T08:55:17.356828+0000","last_active":"2024-04-12T09:03:31.474613+0000","last_peered":"2024-04-12T09:03:31.474613+0000","last_clean":"2024-04-12T09:03:31.474613+0000","last_became_active":"2024-04-12T08:55:17.356374+0000","last_became_peered":"2024-04-12T08:55:17.356374+0000","last_unstale":"2024-04-12T09:03:31.474613+0000","last_undegraded":"2024-04-12T09:03:31.474613+0000","last_fullsized":"2024-04-12T09:03:31.474613+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_clean_scrub_stamp":"2024-04-12T08:39:39.156733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:05:48.388222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.480919+0000","last_change":"2024-04-12T09:03:32.480919+0000","last_active":"2024-04-12T09:03:32.480919+0000","last_peered":"2024-04-12T09:03:32.480919+0000","last_clean":"2024-04-12T09:03:32.480919+0000","last_became_active":"2024-04-12T09:03:32.480457+0000","last_became_peered":"2024-04-12T09:03:32.480457+0000","last_unstale":"2024-04-12T09:03:32.480919+0000","last_undegraded":"2024-04-12T09:03:32.480919+0000","last_fullsized":"2024-04-12T09:03:32.480919+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:15.500566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.471350+0000","last_change":"2024-04-12T08:43:29.892386+0000","last_active":"2024-04-12T09:03:31.471350+0000","last_peered":"2024-04-12T09:03:31.471350+0000","last_clean":"2024-04-12T09:03:31.471350+0000","last_became_active":"2024-04-12T08:43:29.892091+0000","last_became_peered":"2024-04-12T08:43:29.892091+0000","last_unstale":"2024-04-12T09:03:31.471350+0000","last_undegraded":"2024-04-12T09:03:31.471350+0000","last_fullsized":"2024-04-12T09:03:31.471350+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:32:11.122269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":119,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.474606+0000","last_change":"2024-04-12T08:58:56.736406+0000","last_active":"2024-04-12T09:03:31.474606+0000","last_peered":"2024-04-12T09:03:31.474606+0000","last_clean":"2024-04-12T09:03:31.474606+0000","last_became_active":"2024-04-12T08:58:56.735971+0000","last_became_peered":"2024-04-12T08:58:56.735971+0000","last_unstale":"2024-04-12T09:03:31.474606+0000","last_undegraded":"2024-04-12T09:03:31.474606+0000","last_fullsized":"2024-04-12T09:03:31.474606+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:46:54.628137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":120,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.486108+0000","last_change":"2024-04-12T09:03:32.485565+0000","last_active":"2024-04-12T09:03:32.486108+0000","last_peered":"2024-04-12T09:03:32.486108+0000","last_clean":"2024-04-12T09:03:32.486108+0000","last_became_active":"2024-04-12T09:03:32.485263+0000","last_became_peered":"2024-04-12T09:03:32.485263+0000","last_unstale":"2024-04-12T09:03:32.486108+0000","last_undegraded":"2024-04-12T09:03:32.486108+0000","last_fullsized":"2024-04-12T09:03:32.486108+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:07.310701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":130,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.478028+0000","last_change":"2024-04-12T09:02:57.003398+0000","last_active":"2024-04-12T09:03:32.478028+0000","last_peered":"2024-04-12T09:03:32.478028+0000","last_clean":"2024-04-12T09:03:32.478028+0000","last_became_active":"2024-04-12T09:02:57.003081+0000","last_became_peered":"2024-04-12T09:02:57.003081+0000","last_unstale":"2024-04-12T09:03:32.478028+0000","last_undegraded":"2024-04-12T09:03:32.478028+0000","last_fullsized":"2024-04-12T09:03:32.478028+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:50:39.887482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":102,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.475007+0000","last_change":"2024-04-12T08:55:17.424843+0000","last_active":"2024-04-12T09:03:31.475007+0000","last_peered":"2024-04-12T09:03:31.475007+0000","last_clean":"2024-04-12T09:03:31.475007+0000","last_became_active":"2024-04-12T08:55:17.424447+0000","last_became_peered":"2024-04-12T08:55:17.424447+0000","last_unstale":"2024-04-12T09:03:31.475007+0000","last_undegraded":"2024-04-12T09:03:31.475007+0000","last_fullsized":"2024-04-12T09:03:31.475007+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:59:20.792914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":118,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.474047+0000","last_change":"2024-04-12T09:02:57.005550+0000","last_active":"2024-04-12T09:03:32.474047+0000","last_peered":"2024-04-12T09:03:32.474047+0000","last_clean":"2024-04-12T09:03:32.474047+0000","last_became_active":"2024-04-12T09:02:57.005256+0000","last_became_peered":"2024-04-12T09:02:57.005256+0000","last_unstale":"2024-04-12T09:03:32.474047+0000","last_undegraded":"2024-04-12T09:03:32.474047+0000","last_fullsized":"2024-04-12T09:03:32.474047+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:27:20.156392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":112,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.476659+0000","last_change":"2024-04-12T08:55:17.359973+0000","last_active":"2024-04-12T09:03:32.476659+0000","last_peered":"2024-04-12T09:03:32.476659+0000","last_clean":"2024-04-12T09:03:32.476659+0000","last_became_active":"2024-04-12T08:55:17.357844+0000","last_became_peered":"2024-04-12T08:55:17.357844+0000","last_unstale":"2024-04-12T09:03:32.476659+0000","last_undegraded":"2024-04-12T09:03:32.476659+0000","last_fullsized":"2024-04-12T09:03:32.476659+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:57:50.740134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":71,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.484186+0000","last_change":"2024-04-12T09:03:32.484186+0000","last_active":"2024-04-12T09:03:32.484186+0000","last_peered":"2024-04-12T09:03:32.484186+0000","last_clean":"2024-04-12T09:03:32.484186+0000","last_became_active":"2024-04-12T09:03:32.482981+0000","last_became_peered":"2024-04-12T09:03:32.482981+0000","last_unstale":"2024-04-12T09:03:32.484186+0000","last_undegraded":"2024-04-12T09:03:32.484186+0000","last_fullsized":"2024-04-12T09:03:32.484186+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:50:05.928185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":109,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473656+0000","last_change":"2024-04-12T08:58:56.737968+0000","last_active":"2024-04-12T09:03:32.473656+0000","last_peered":"2024-04-12T09:03:32.473656+0000","last_clean":"2024-04-12T09:03:32.473656+0000","last_became_active":"2024-04-12T08:58:56.737579+0000","last_became_peered":"2024-04-12T08:58:56.737579+0000","last_unstale":"2024-04-12T09:03:32.473656+0000","last_undegraded":"2024-04-12T09:03:32.473656+0000","last_fullsized":"2024-04-12T09:03:32.473656+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:09:45.836715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.484123+0000","last_change":"2024-04-12T09:03:32.484123+0000","last_active":"2024-04-12T09:03:32.484123+0000","last_peered":"2024-04-12T09:03:32.484123+0000","last_clean":"2024-04-12T09:03:32.484123+0000","last_became_active":"2024-04-12T09:03:32.483052+0000","last_became_peered":"2024-04-12T09:03:32.483052+0000","last_unstale":"2024-04-12T09:03:32.484123+0000","last_undegraded":"2024-04-12T09:03:32.484123+0000","last_fullsized":"2024-04-12T09:03:32.484123+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:01:06.479551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":102,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.471135+0000","last_change":"2024-04-12T09:02:57.007543+0000","last_active":"2024-04-12T09:03:31.471135+0000","last_peered":"2024-04-12T09:03:31.471135+0000","last_clean":"2024-04-12T09:03:31.471135+0000","last_became_active":"2024-04-12T09:02:57.007234+0000","last_became_peered":"2024-04-12T09:02:57.007234+0000","last_unstale":"2024-04-12T09:03:31.471135+0000","last_undegraded":"2024-04-12T09:03:31.471135+0000","last_fullsized":"2024-04-12T09:03:31.471135+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:35.310595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":130,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.477688+0000","last_change":"2024-04-12T09:02:57.005094+0000","last_active":"2024-04-12T09:03:32.477688+0000","last_peered":"2024-04-12T09:03:32.477688+0000","last_clean":"2024-04-12T09:03:32.477688+0000","last_became_active":"2024-04-12T09:02:57.004691+0000","last_became_peered":"2024-04-12T09:02:57.004691+0000","last_unstale":"2024-04-12T09:03:32.477688+0000","last_undegraded":"2024-04-12T09:03:32.477688+0000","last_fullsized":"2024-04-12T09:03:32.477688+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:06:07.486319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.466794+0000","last_change":"2024-04-12T08:43:28.365741+0000","last_active":"2024-04-12T09:03:31.466794+0000","last_peered":"2024-04-12T09:03:31.466794+0000","last_clean":"2024-04-12T09:03:31.466794+0000","last_became_active":"2024-04-12T08:43:28.365425+0000","last_became_peered":"2024-04-12T08:43:28.365425+0000","last_unstale":"2024-04-12T09:03:31.466794+0000","last_undegraded":"2024-04-12T09:03:31.466794+0000","last_fullsized":"2024-04-12T09:03:31.466794+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:13:45.303498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":114,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.476716+0000","last_change":"2024-04-12T08:55:17.355386+0000","last_active":"2024-04-12T09:03:32.476716+0000","last_peered":"2024-04-12T09:03:32.476716+0000","last_clean":"2024-04-12T09:03:32.476716+0000","last_became_active":"2024-04-12T08:55:17.353579+0000","last_became_peered":"2024-04-12T08:55:17.353579+0000","last_unstale":"2024-04-12T09:03:32.476716+0000","last_undegraded":"2024-04-12T09:03:32.476716+0000","last_fullsized":"2024-04-12T09:03:32.476716+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:56:14.416234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":95,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479337+0000","last_change":"2024-04-12T09:03:32.479337+0000","last_active":"2024-04-12T09:03:32.479337+0000","last_peered":"2024-04-12T09:03:32.479337+0000","last_clean":"2024-04-12T09:03:32.479337+0000","last_became_active":"2024-04-12T09:03:32.478916+0000","last_became_peered":"2024-04-12T09:03:32.478916+0000","last_unstale":"2024-04-12T09:03:32.479337+0000","last_undegraded":"2024-04-12T09:03:32.479337+0000","last_fullsized":"2024-04-12T09:03:32.479337+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:54:08.702850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":129,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469699+0000","last_change":"2024-04-12T08:58:56.736545+0000","last_active":"2024-04-12T09:03:31.469699+0000","last_peered":"2024-04-12T09:03:31.469699+0000","last_clean":"2024-04-12T09:03:31.469699+0000","last_became_active":"2024-04-12T08:58:56.736182+0000","last_became_peered":"2024-04-12T08:58:56.736182+0000","last_unstale":"2024-04-12T09:03:31.469699+0000","last_undegraded":"2024-04-12T09:03:31.469699+0000","last_fullsized":"2024-04-12T09:03:31.469699+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:39:18.004066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.484285+0000","last_change":"2024-04-12T09:03:32.484285+0000","last_active":"2024-04-12T09:03:32.484285+0000","last_peered":"2024-04-12T09:03:32.484285+0000","last_clean":"2024-04-12T09:03:32.484285+0000","last_became_active":"2024-04-12T09:03:32.483889+0000","last_became_peered":"2024-04-12T09:03:32.483889+0000","last_unstale":"2024-04-12T09:03:32.484285+0000","last_undegraded":"2024-04-12T09:03:32.484285+0000","last_fullsized":"2024-04-12T09:03:32.484285+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:13:29.920796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":84,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.470815+0000","last_change":"2024-04-12T08:43:28.137332+0000","last_active":"2024-04-12T09:03:31.470815+0000","last_peered":"2024-04-12T09:03:31.470815+0000","last_clean":"2024-04-12T09:03:31.470815+0000","last_became_active":"2024-04-12T08:43:28.136749+0000","last_became_peered":"2024-04-12T08:43:28.136749+0000","last_unstale":"2024-04-12T09:03:31.470815+0000","last_undegraded":"2024-04-12T09:03:31.470815+0000","last_fullsized":"2024-04-12T09:03:31.470815+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:20:09.223627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.483824+0000","last_change":"2024-04-12T09:03:32.483824+0000","last_active":"2024-04-12T09:03:32.483824+0000","last_peered":"2024-04-12T09:03:32.483824+0000","last_clean":"2024-04-12T09:03:32.483824+0000","last_became_active":"2024-04-12T09:03:32.483434+0000","last_became_peered":"2024-04-12T09:03:32.483434+0000","last_unstale":"2024-04-12T09:03:32.483824+0000","last_undegraded":"2024-04-12T09:03:32.483824+0000","last_fullsized":"2024-04-12T09:03:32.483824+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:13:33.249176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":107,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.481382+0000","last_change":"2024-04-12T09:03:32.481382+0000","last_active":"2024-04-12T09:03:32.481382+0000","last_peered":"2024-04-12T09:03:32.481382+0000","last_clean":"2024-04-12T09:03:32.481382+0000","last_became_active":"2024-04-12T09:03:32.480957+0000","last_became_peered":"2024-04-12T09:03:32.480957+0000","last_unstale":"2024-04-12T09:03:32.481382+0000","last_undegraded":"2024-04-12T09:03:32.481382+0000","last_fullsized":"2024-04-12T09:03:32.481382+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:06:39.854442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":88,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469500+0000","last_change":"2024-04-12T09:02:57.004852+0000","last_active":"2024-04-12T09:03:31.469500+0000","last_peered":"2024-04-12T09:03:31.469500+0000","last_clean":"2024-04-12T09:03:31.469500+0000","last_became_active":"2024-04-12T09:02:57.004532+0000","last_became_peered":"2024-04-12T09:02:57.004532+0000","last_unstale":"2024-04-12T09:03:31.469500+0000","last_undegraded":"2024-04-12T09:03:31.469500+0000","last_fullsized":"2024-04-12T09:03:31.469500+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:08:22.187536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":124,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.476047+0000","last_change":"2024-04-12T09:02:57.005618+0000","last_active":"2024-04-12T09:03:32.476047+0000","last_peered":"2024-04-12T09:03:32.476047+0000","last_clean":"2024-04-12T09:03:32.476047+0000","last_became_active":"2024-04-12T09:02:57.005238+0000","last_became_peered":"2024-04-12T09:02:57.005238+0000","last_unstale":"2024-04-12T09:03:32.476047+0000","last_undegraded":"2024-04-12T09:03:32.476047+0000","last_fullsized":"2024-04-12T09:03:32.476047+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:23:01.772101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.466794+0000","last_change":"2024-04-12T09:02:57.000721+0000","last_active":"2024-04-12T09:03:31.466794+0000","last_peered":"2024-04-12T09:03:31.466794+0000","last_clean":"2024-04-12T09:03:31.466794+0000","last_became_active":"2024-04-12T09:02:57.000236+0000","last_became_peered":"2024-04-12T09:02:57.000236+0000","last_unstale":"2024-04-12T09:03:31.466794+0000","last_undegraded":"2024-04-12T09:03:31.466794+0000","last_fullsized":"2024-04-12T09:03:31.466794+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:00:46.597355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4834,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15383,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201557,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299652,"kb_used_data":2288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445500,"statfs":{"total":95995035648,"available":95688192000,"internally_reserved":0,"allocated":2342912,"data_stored":1501158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463193,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303724,"kb_used_data":2272,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93441428,"statfs":{"total":95995035648,"available":95684022272,"internally_reserved":0,"allocated":2326528,"data_stored":1500564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":116,"seq":498216206405,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":181992,"kb_used_data":1684,"kb_used_omap":5,"kb_used_meta":180282,"kb_avail":93563160,"statfs":{"total":95995035648,"available":95808675840,"internally_reserved":0,"allocated":1724416,"data_stored":905823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5934,"internal_metadata":184608978},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":95,"seq":408021893233,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":262404,"kb_used_data":2288,"kb_used_omap":5,"kb_used_meta":260090,"kb_avail":93482748,"statfs":{"total":95995035648,"available":95726333952,"internally_reserved":0,"allocated":2342912,"data_stored":1501776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6054,"internal_metadata":266332250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":133,"seq":571230650389,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":61748,"kb_used_data":1696,"kb_used_omap":6,"kb_used_meta":60025,"kb_avail":93683404,"statfs":{"total":95995035648,"available":95931805696,"internally_reserved":0,"allocated":1736704,"data_stored":906851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6732,"internal_metadata":61466036},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":138,"seq":592705486862,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":80488,"kb_used_data":1692,"kb_used_omap":4,"kb_used_meta":78779,"kb_avail":93664664,"statfs":{"total":95995035648,"available":95912615936,"internally_reserved":0,"allocated":1732608,"data_stored":906337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4638,"internal_metadata":80670178},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986467,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599732,"kb_used_data":1696,"kb_used_omap":1,"kb_used_meta":598014,"kb_avail":93145420,"statfs":{"total":95995035648,"available":95380910080,"internally_reserved":0,"allocated":1736704,"data_stored":906283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612366680},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724830,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907308,"kb_used_data":1696,"kb_used_omap":1,"kb_used_meta":905598,"kb_avail":92837844,"statfs":{"total":95995035648,"available":95065952256,"internally_reserved":0,"allocated":1736704,"data_stored":906875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":927332696},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4350,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4230,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2934,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T09:04:33.073 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T09:04:33.331 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:33.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:04:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:04:33] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.43.0" 2024-04-12T09:04:33.741 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:33 smithi090 ceph-mon[99575]: from='client.35361 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T09:04:33.741 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:33 smithi090 ceph-mon[95814]: from='client.35361 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T09:04:33.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:33 smithi114 ceph-mon[89482]: from='client.35361 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T09:04:34.759 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:34 smithi090 ceph-mon[99575]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:34.759 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:34 smithi090 ceph-mon[95814]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:34 smithi114 ceph-mon[89482]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:35.191 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:04:35.192 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-12T09:04:35.890 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":518,"stamp":"2024-04-12T09:04:33.392302+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":1775938,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2697048,"kb_used_data":15312,"kb_used_omap":29,"kb_used_meta":2681570,"kb_avail":747264168,"statfs":{"total":767960285184,"available":765198508032,"internally_reserved":0,"allocated":15679488,"data_stored":9035667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30174,"internal_metadata":2745928226},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002789"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":117,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.466882+0000","last_change":"2024-04-12T09:02:57.000425+0000","last_active":"2024-04-12T09:03:31.466882+0000","last_peered":"2024-04-12T09:03:31.466882+0000","last_clean":"2024-04-12T09:03:31.466882+0000","last_became_active":"2024-04-12T09:02:56.999444+0000","last_became_peered":"2024-04-12T09:02:56.999444+0000","last_unstale":"2024-04-12T09:03:31.466882+0000","last_undegraded":"2024-04-12T09:03:31.466882+0000","last_fullsized":"2024-04-12T09:03:31.466882+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:43:56.538903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":89,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479711+0000","last_change":"2024-04-12T09:03:32.479711+0000","last_active":"2024-04-12T09:03:32.479711+0000","last_peered":"2024-04-12T09:03:32.479711+0000","last_clean":"2024-04-12T09:03:32.479711+0000","last_became_active":"2024-04-12T09:03:32.478500+0000","last_became_peered":"2024-04-12T09:03:32.478500+0000","last_unstale":"2024-04-12T09:03:32.479711+0000","last_undegraded":"2024-04-12T09:03:32.479711+0000","last_fullsized":"2024-04-12T09:03:32.479711+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:12:34.581151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":122,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473038+0000","last_change":"2024-04-12T08:58:56.743931+0000","last_active":"2024-04-12T09:03:32.473038+0000","last_peered":"2024-04-12T09:03:32.473038+0000","last_clean":"2024-04-12T09:03:32.473038+0000","last_became_active":"2024-04-12T08:58:56.740909+0000","last_became_peered":"2024-04-12T08:58:56.740909+0000","last_unstale":"2024-04-12T09:03:32.473038+0000","last_undegraded":"2024-04-12T09:03:32.473038+0000","last_fullsized":"2024-04-12T09:03:32.473038+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:54:05.956455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":121,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.486300+0000","last_change":"2024-04-12T09:03:32.486300+0000","last_active":"2024-04-12T09:03:32.486300+0000","last_peered":"2024-04-12T09:03:32.486300+0000","last_clean":"2024-04-12T09:03:32.486300+0000","last_became_active":"2024-04-12T09:03:32.485911+0000","last_became_peered":"2024-04-12T09:03:32.485911+0000","last_unstale":"2024-04-12T09:03:32.486300+0000","last_undegraded":"2024-04-12T09:03:32.486300+0000","last_fullsized":"2024-04-12T09:03:32.486300+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:02:35.257943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":107,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.483915+0000","last_change":"2024-04-12T09:03:32.483915+0000","last_active":"2024-04-12T09:03:32.483915+0000","last_peered":"2024-04-12T09:03:32.483915+0000","last_clean":"2024-04-12T09:03:32.483915+0000","last_became_active":"2024-04-12T09:03:32.483345+0000","last_became_peered":"2024-04-12T09:03:32.483345+0000","last_unstale":"2024-04-12T09:03:32.483915+0000","last_undegraded":"2024-04-12T09:03:32.483915+0000","last_fullsized":"2024-04-12T09:03:32.483915+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:42:17.502077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":97,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.471313+0000","last_change":"2024-04-12T08:58:56.774042+0000","last_active":"2024-04-12T09:03:31.471313+0000","last_peered":"2024-04-12T09:03:31.471313+0000","last_clean":"2024-04-12T09:03:31.471313+0000","last_became_active":"2024-04-12T08:58:56.773622+0000","last_became_peered":"2024-04-12T08:58:56.773622+0000","last_unstale":"2024-04-12T09:03:31.471313+0000","last_undegraded":"2024-04-12T09:03:31.471313+0000","last_fullsized":"2024-04-12T09:03:31.471313+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:08:11.813135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479868+0000","last_change":"2024-04-12T09:03:32.479868+0000","last_active":"2024-04-12T09:03:32.479868+0000","last_peered":"2024-04-12T09:03:32.479868+0000","last_clean":"2024-04-12T09:03:32.479868+0000","last_became_active":"2024-04-12T09:03:32.479576+0000","last_became_peered":"2024-04-12T09:03:32.479576+0000","last_unstale":"2024-04-12T09:03:32.479868+0000","last_undegraded":"2024-04-12T09:03:32.479868+0000","last_fullsized":"2024-04-12T09:03:32.479868+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:28:47.455110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":89,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479067+0000","last_change":"2024-04-12T09:03:32.479067+0000","last_active":"2024-04-12T09:03:32.479067+0000","last_peered":"2024-04-12T09:03:32.479067+0000","last_clean":"2024-04-12T09:03:32.479067+0000","last_became_active":"2024-04-12T09:03:32.478746+0000","last_became_peered":"2024-04-12T09:03:32.478746+0000","last_unstale":"2024-04-12T09:03:32.479067+0000","last_undegraded":"2024-04-12T09:03:32.479067+0000","last_fullsized":"2024-04-12T09:03:32.479067+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:40:04.214931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469850+0000","last_change":"2024-04-12T08:58:56.736460+0000","last_active":"2024-04-12T09:03:31.469850+0000","last_peered":"2024-04-12T09:03:31.469850+0000","last_clean":"2024-04-12T09:03:31.469850+0000","last_became_active":"2024-04-12T08:58:56.736126+0000","last_became_peered":"2024-04-12T08:58:56.736126+0000","last_unstale":"2024-04-12T09:03:31.469850+0000","last_undegraded":"2024-04-12T09:03:31.469850+0000","last_fullsized":"2024-04-12T09:03:31.469850+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:29:35.656864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.482168+0000","last_change":"2024-04-12T09:03:32.482168+0000","last_active":"2024-04-12T09:03:32.482168+0000","last_peered":"2024-04-12T09:03:32.482168+0000","last_clean":"2024-04-12T09:03:32.482168+0000","last_became_active":"2024-04-12T09:03:32.481615+0000","last_became_peered":"2024-04-12T09:03:32.481615+0000","last_unstale":"2024-04-12T09:03:32.482168+0000","last_undegraded":"2024-04-12T09:03:32.482168+0000","last_fullsized":"2024-04-12T09:03:32.482168+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:52:49.384832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":121,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.486344+0000","last_change":"2024-04-12T09:03:32.486344+0000","last_active":"2024-04-12T09:03:32.486344+0000","last_peered":"2024-04-12T09:03:32.486344+0000","last_clean":"2024-04-12T09:03:32.486344+0000","last_became_active":"2024-04-12T09:03:32.485987+0000","last_became_peered":"2024-04-12T09:03:32.485987+0000","last_unstale":"2024-04-12T09:03:32.486344+0000","last_undegraded":"2024-04-12T09:03:32.486344+0000","last_fullsized":"2024-04-12T09:03:32.486344+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:44:10.209457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":119,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473223+0000","last_change":"2024-04-12T09:02:56.999939+0000","last_active":"2024-04-12T09:03:32.473223+0000","last_peered":"2024-04-12T09:03:32.473223+0000","last_clean":"2024-04-12T09:03:32.473223+0000","last_became_active":"2024-04-12T09:02:56.999624+0000","last_became_peered":"2024-04-12T09:02:56.999624+0000","last_unstale":"2024-04-12T09:03:32.473223+0000","last_undegraded":"2024-04-12T09:03:32.473223+0000","last_fullsized":"2024-04-12T09:03:32.473223+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:40:02.678154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":107,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.477074+0000","last_change":"2024-04-12T08:55:18.487199+0000","last_active":"2024-04-12T09:03:32.477074+0000","last_peered":"2024-04-12T09:03:32.477074+0000","last_clean":"2024-04-12T09:03:32.477074+0000","last_became_active":"2024-04-12T08:55:18.391671+0000","last_became_peered":"2024-04-12T08:55:18.391671+0000","last_unstale":"2024-04-12T09:03:32.477074+0000","last_undegraded":"2024-04-12T09:03:32.477074+0000","last_fullsized":"2024-04-12T09:03:32.477074+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:41:36.634168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":98,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.467663+0000","last_change":"2024-04-12T08:58:56.736878+0000","last_active":"2024-04-12T09:03:31.467663+0000","last_peered":"2024-04-12T09:03:31.467663+0000","last_clean":"2024-04-12T09:03:31.467663+0000","last_became_active":"2024-04-12T08:58:56.736472+0000","last_became_peered":"2024-04-12T08:58:56.736472+0000","last_unstale":"2024-04-12T09:03:31.467663+0000","last_undegraded":"2024-04-12T09:03:31.467663+0000","last_fullsized":"2024-04-12T09:03:31.467663+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:48:57.668138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":100,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.474796+0000","last_change":"2024-04-12T09:02:56.999551+0000","last_active":"2024-04-12T09:03:31.474796+0000","last_peered":"2024-04-12T09:03:31.474796+0000","last_clean":"2024-04-12T09:03:31.474796+0000","last_became_active":"2024-04-12T09:02:56.999117+0000","last_became_peered":"2024-04-12T09:02:56.999117+0000","last_unstale":"2024-04-12T09:03:31.474796+0000","last_undegraded":"2024-04-12T09:03:31.474796+0000","last_fullsized":"2024-04-12T09:03:31.474796+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:20:49.562177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.482027+0000","last_change":"2024-04-12T09:03:32.482027+0000","last_active":"2024-04-12T09:03:32.482027+0000","last_peered":"2024-04-12T09:03:32.482027+0000","last_clean":"2024-04-12T09:03:32.482027+0000","last_became_active":"2024-04-12T09:03:32.481606+0000","last_became_peered":"2024-04-12T09:03:32.481606+0000","last_unstale":"2024-04-12T09:03:32.482027+0000","last_undegraded":"2024-04-12T09:03:32.482027+0000","last_fullsized":"2024-04-12T09:03:32.482027+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:20:17.182218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":119,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.485440+0000","last_change":"2024-04-12T09:03:32.485440+0000","last_active":"2024-04-12T09:03:32.485440+0000","last_peered":"2024-04-12T09:03:32.485440+0000","last_clean":"2024-04-12T09:03:32.485440+0000","last_became_active":"2024-04-12T09:03:32.485197+0000","last_became_peered":"2024-04-12T09:03:32.485197+0000","last_unstale":"2024-04-12T09:03:32.485440+0000","last_undegraded":"2024-04-12T09:03:32.485440+0000","last_fullsized":"2024-04-12T09:03:32.485440+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:16:10.437274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479277+0000","last_change":"2024-04-12T09:03:32.479277+0000","last_active":"2024-04-12T09:03:32.479277+0000","last_peered":"2024-04-12T09:03:32.479277+0000","last_clean":"2024-04-12T09:03:32.479277+0000","last_became_active":"2024-04-12T09:03:32.478957+0000","last_became_peered":"2024-04-12T09:03:32.478957+0000","last_unstale":"2024-04-12T09:03:32.479277+0000","last_undegraded":"2024-04-12T09:03:32.479277+0000","last_fullsized":"2024-04-12T09:03:32.479277+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:01:32.853192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.473836+0000","last_change":"2024-04-12T08:43:28.133594+0000","last_active":"2024-04-12T09:03:31.473836+0000","last_peered":"2024-04-12T09:03:31.473836+0000","last_clean":"2024-04-12T09:03:31.473836+0000","last_became_active":"2024-04-12T08:43:28.133305+0000","last_became_peered":"2024-04-12T08:43:28.133305+0000","last_unstale":"2024-04-12T09:03:31.473836+0000","last_undegraded":"2024-04-12T09:03:31.473836+0000","last_fullsized":"2024-04-12T09:03:31.473836+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:09:26.576479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":120,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.486054+0000","last_change":"2024-04-12T09:03:32.486054+0000","last_active":"2024-04-12T09:03:32.486054+0000","last_peered":"2024-04-12T09:03:32.486054+0000","last_clean":"2024-04-12T09:03:32.486054+0000","last_became_active":"2024-04-12T09:03:32.485729+0000","last_became_peered":"2024-04-12T09:03:32.485729+0000","last_unstale":"2024-04-12T09:03:32.486054+0000","last_undegraded":"2024-04-12T09:03:32.486054+0000","last_fullsized":"2024-04-12T09:03:32.486054+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:07:13.286240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":106,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.471382+0000","last_change":"2024-04-12T08:55:17.353074+0000","last_active":"2024-04-12T09:03:31.471382+0000","last_peered":"2024-04-12T09:03:31.471382+0000","last_clean":"2024-04-12T09:03:31.471382+0000","last_became_active":"2024-04-12T08:55:17.352780+0000","last_became_peered":"2024-04-12T08:55:17.352780+0000","last_unstale":"2024-04-12T09:03:31.471382+0000","last_undegraded":"2024-04-12T09:03:31.471382+0000","last_fullsized":"2024-04-12T09:03:31.471382+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:29:03.405163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.481571+0000","last_change":"2024-04-12T09:03:32.481571+0000","last_active":"2024-04-12T09:03:32.481571+0000","last_peered":"2024-04-12T09:03:32.481571+0000","last_clean":"2024-04-12T09:03:32.481571+0000","last_became_active":"2024-04-12T09:03:32.481207+0000","last_became_peered":"2024-04-12T09:03:32.481207+0000","last_unstale":"2024-04-12T09:03:32.481571+0000","last_undegraded":"2024-04-12T09:03:32.481571+0000","last_fullsized":"2024-04-12T09:03:32.481571+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:30:16.040167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":99,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.475837+0000","last_change":"2024-04-12T08:55:17.426384+0000","last_active":"2024-04-12T09:03:31.475837+0000","last_peered":"2024-04-12T09:03:31.475837+0000","last_clean":"2024-04-12T09:03:31.475837+0000","last_became_active":"2024-04-12T08:55:17.425999+0000","last_became_peered":"2024-04-12T08:55:17.425999+0000","last_unstale":"2024-04-12T09:03:31.475837+0000","last_undegraded":"2024-04-12T09:03:31.475837+0000","last_fullsized":"2024-04-12T09:03:31.475837+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:10:35.822055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469441+0000","last_change":"2024-04-12T08:43:28.866806+0000","last_active":"2024-04-12T09:03:31.469441+0000","last_peered":"2024-04-12T09:03:31.469441+0000","last_clean":"2024-04-12T09:03:31.469441+0000","last_became_active":"2024-04-12T08:43:28.866472+0000","last_became_peered":"2024-04-12T08:43:28.866472+0000","last_unstale":"2024-04-12T09:03:31.469441+0000","last_undegraded":"2024-04-12T09:03:31.469441+0000","last_fullsized":"2024-04-12T09:03:31.469441+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:23.104376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":100,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.475388+0000","last_change":"2024-04-12T08:58:56.737605+0000","last_active":"2024-04-12T09:03:31.475388+0000","last_peered":"2024-04-12T09:03:31.475388+0000","last_clean":"2024-04-12T09:03:31.475388+0000","last_became_active":"2024-04-12T08:58:56.736799+0000","last_became_peered":"2024-04-12T08:58:56.736799+0000","last_unstale":"2024-04-12T09:03:31.475388+0000","last_undegraded":"2024-04-12T09:03:31.475388+0000","last_fullsized":"2024-04-12T09:03:31.475388+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:45.117695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.480130+0000","last_change":"2024-04-12T09:03:32.480130+0000","last_active":"2024-04-12T09:03:32.480130+0000","last_peered":"2024-04-12T09:03:32.480130+0000","last_clean":"2024-04-12T09:03:32.480130+0000","last_became_active":"2024-04-12T09:03:32.479831+0000","last_became_peered":"2024-04-12T09:03:32.479831+0000","last_unstale":"2024-04-12T09:03:32.480130+0000","last_undegraded":"2024-04-12T09:03:32.480130+0000","last_fullsized":"2024-04-12T09:03:32.480130+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:50:04.535521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":82,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.474915+0000","last_change":"2024-04-12T08:43:28.133358+0000","last_active":"2024-04-12T09:03:31.474915+0000","last_peered":"2024-04-12T09:03:31.474915+0000","last_clean":"2024-04-12T09:03:31.474915+0000","last_became_active":"2024-04-12T08:43:28.133112+0000","last_became_peered":"2024-04-12T08:43:28.133112+0000","last_unstale":"2024-04-12T09:03:31.474915+0000","last_undegraded":"2024-04-12T09:03:31.474915+0000","last_fullsized":"2024-04-12T09:03:31.474915+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:29:38.251048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":97,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.470947+0000","last_change":"2024-04-12T08:58:56.773685+0000","last_active":"2024-04-12T09:03:31.470947+0000","last_peered":"2024-04-12T09:03:31.470947+0000","last_clean":"2024-04-12T09:03:31.470947+0000","last_became_active":"2024-04-12T08:58:56.773314+0000","last_became_peered":"2024-04-12T08:58:56.773314+0000","last_unstale":"2024-04-12T09:03:31.470947+0000","last_undegraded":"2024-04-12T09:03:31.470947+0000","last_fullsized":"2024-04-12T09:03:31.470947+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:29:42.013427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.477270+0000","last_change":"2024-04-12T09:03:32.477270+0000","last_active":"2024-04-12T09:03:32.477270+0000","last_peered":"2024-04-12T09:03:32.477270+0000","last_clean":"2024-04-12T09:03:32.477270+0000","last_became_active":"2024-04-12T09:03:32.476901+0000","last_became_peered":"2024-04-12T09:03:32.476901+0000","last_unstale":"2024-04-12T09:03:32.477270+0000","last_undegraded":"2024-04-12T09:03:32.477270+0000","last_fullsized":"2024-04-12T09:03:32.477270+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:38:19.824709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":121,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.476741+0000","last_change":"2024-04-12T08:58:56.744712+0000","last_active":"2024-04-12T09:03:32.476741+0000","last_peered":"2024-04-12T09:03:32.476741+0000","last_clean":"2024-04-12T09:03:32.476741+0000","last_became_active":"2024-04-12T08:58:56.744396+0000","last_became_peered":"2024-04-12T08:58:56.744396+0000","last_unstale":"2024-04-12T09:03:32.476741+0000","last_undegraded":"2024-04-12T09:03:32.476741+0000","last_fullsized":"2024-04-12T09:03:32.476741+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:14:40.787811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":71,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.484437+0000","last_change":"2024-04-12T09:03:32.484437+0000","last_active":"2024-04-12T09:03:32.484437+0000","last_peered":"2024-04-12T09:03:32.484437+0000","last_clean":"2024-04-12T09:03:32.484437+0000","last_became_active":"2024-04-12T09:03:32.483444+0000","last_became_peered":"2024-04-12T09:03:32.483444+0000","last_unstale":"2024-04-12T09:03:32.484437+0000","last_undegraded":"2024-04-12T09:03:32.484437+0000","last_fullsized":"2024-04-12T09:03:32.484437+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:19.582083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.484024+0000","last_change":"2024-04-12T09:03:32.484024+0000","last_active":"2024-04-12T09:03:32.484024+0000","last_peered":"2024-04-12T09:03:32.484024+0000","last_clean":"2024-04-12T09:03:32.484024+0000","last_became_active":"2024-04-12T09:03:32.483693+0000","last_became_peered":"2024-04-12T09:03:32.483693+0000","last_unstale":"2024-04-12T09:03:32.484024+0000","last_undegraded":"2024-04-12T09:03:32.484024+0000","last_fullsized":"2024-04-12T09:03:32.484024+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:45:51.457544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":122,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.470249+0000","last_change":"2024-04-12T08:58:56.774315+0000","last_active":"2024-04-12T09:03:31.470249+0000","last_peered":"2024-04-12T09:03:31.470249+0000","last_clean":"2024-04-12T09:03:31.470249+0000","last_became_active":"2024-04-12T08:58:56.774000+0000","last_became_peered":"2024-04-12T08:58:56.774000+0000","last_unstale":"2024-04-12T09:03:31.470249+0000","last_undegraded":"2024-04-12T09:03:31.470249+0000","last_fullsized":"2024-04-12T09:03:31.470249+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:19:25.327127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.467100+0000","last_change":"2024-04-12T09:02:56.999564+0000","last_active":"2024-04-12T09:03:31.467100+0000","last_peered":"2024-04-12T09:03:31.467100+0000","last_clean":"2024-04-12T09:03:31.467100+0000","last_became_active":"2024-04-12T09:02:56.999215+0000","last_became_peered":"2024-04-12T09:02:56.999215+0000","last_unstale":"2024-04-12T09:03:31.467100+0000","last_undegraded":"2024-04-12T09:03:31.467100+0000","last_fullsized":"2024-04-12T09:03:31.467100+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:51:28.380649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.466872+0000","last_change":"2024-04-12T09:02:56.999590+0000","last_active":"2024-04-12T09:03:31.466872+0000","last_peered":"2024-04-12T09:03:31.466872+0000","last_clean":"2024-04-12T09:03:31.466872+0000","last_became_active":"2024-04-12T09:02:56.999119+0000","last_became_peered":"2024-04-12T09:02:56.999119+0000","last_unstale":"2024-04-12T09:03:31.466872+0000","last_undegraded":"2024-04-12T09:03:31.466872+0000","last_fullsized":"2024-04-12T09:03:31.466872+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:44:45.124730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.467149+0000","last_change":"2024-04-12T08:58:56.738328+0000","last_active":"2024-04-12T09:03:31.467149+0000","last_peered":"2024-04-12T09:03:31.467149+0000","last_clean":"2024-04-12T09:03:31.467149+0000","last_became_active":"2024-04-12T08:58:56.737689+0000","last_became_peered":"2024-04-12T08:58:56.737689+0000","last_unstale":"2024-04-12T09:03:31.467149+0000","last_undegraded":"2024-04-12T09:03:31.467149+0000","last_fullsized":"2024-04-12T09:03:31.467149+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:27:04.300888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.482407+0000","last_change":"2024-04-12T09:03:32.482407+0000","last_active":"2024-04-12T09:03:32.482407+0000","last_peered":"2024-04-12T09:03:32.482407+0000","last_clean":"2024-04-12T09:03:32.482407+0000","last_became_active":"2024-04-12T09:03:32.481981+0000","last_became_peered":"2024-04-12T09:03:32.481981+0000","last_unstale":"2024-04-12T09:03:32.482407+0000","last_undegraded":"2024-04-12T09:03:32.482407+0000","last_fullsized":"2024-04-12T09:03:32.482407+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:28:43.592976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":95,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.478444+0000","last_change":"2024-04-12T09:03:32.478444+0000","last_active":"2024-04-12T09:03:32.478444+0000","last_peered":"2024-04-12T09:03:32.478444+0000","last_clean":"2024-04-12T09:03:32.478444+0000","last_became_active":"2024-04-12T09:03:32.478110+0000","last_became_peered":"2024-04-12T09:03:32.478110+0000","last_unstale":"2024-04-12T09:03:32.478444+0000","last_undegraded":"2024-04-12T09:03:32.478444+0000","last_fullsized":"2024-04-12T09:03:32.478444+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:57:39.815554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":118,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473714+0000","last_change":"2024-04-12T08:58:56.742998+0000","last_active":"2024-04-12T09:03:32.473714+0000","last_peered":"2024-04-12T09:03:32.473714+0000","last_clean":"2024-04-12T09:03:32.473714+0000","last_became_active":"2024-04-12T08:58:56.742632+0000","last_became_peered":"2024-04-12T08:58:56.742632+0000","last_unstale":"2024-04-12T09:03:32.473714+0000","last_undegraded":"2024-04-12T09:03:32.473714+0000","last_fullsized":"2024-04-12T09:03:32.473714+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:27:20.286312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.477229+0000","last_change":"2024-04-12T09:03:32.477229+0000","last_active":"2024-04-12T09:03:32.477229+0000","last_peered":"2024-04-12T09:03:32.477229+0000","last_clean":"2024-04-12T09:03:32.477229+0000","last_became_active":"2024-04-12T09:03:32.476902+0000","last_became_peered":"2024-04-12T09:03:32.476902+0000","last_unstale":"2024-04-12T09:03:32.477229+0000","last_undegraded":"2024-04-12T09:03:32.477229+0000","last_fullsized":"2024-04-12T09:03:32.477229+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:29:32.950731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":110,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473276+0000","last_change":"2024-04-12T08:58:56.739554+0000","last_active":"2024-04-12T09:03:32.473276+0000","last_peered":"2024-04-12T09:03:32.473276+0000","last_clean":"2024-04-12T09:03:32.473276+0000","last_became_active":"2024-04-12T08:58:56.737612+0000","last_became_peered":"2024-04-12T08:58:56.737612+0000","last_unstale":"2024-04-12T09:03:32.473276+0000","last_undegraded":"2024-04-12T09:03:32.473276+0000","last_fullsized":"2024-04-12T09:03:32.473276+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:30.076059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":103,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469512+0000","last_change":"2024-04-12T08:58:56.775286+0000","last_active":"2024-04-12T09:03:31.469512+0000","last_peered":"2024-04-12T09:03:31.469512+0000","last_clean":"2024-04-12T09:03:31.469512+0000","last_became_active":"2024-04-12T08:58:56.774900+0000","last_became_peered":"2024-04-12T08:58:56.774900+0000","last_unstale":"2024-04-12T09:03:31.469512+0000","last_undegraded":"2024-04-12T09:03:31.469512+0000","last_fullsized":"2024-04-12T09:03:31.469512+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:02:51.435526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.467547+0000","last_change":"2024-04-12T08:55:17.354270+0000","last_active":"2024-04-12T09:03:31.467547+0000","last_peered":"2024-04-12T09:03:31.467547+0000","last_clean":"2024-04-12T09:03:31.467547+0000","last_became_active":"2024-04-12T08:55:17.353940+0000","last_became_peered":"2024-04-12T08:55:17.353940+0000","last_unstale":"2024-04-12T09:03:31.467547+0000","last_undegraded":"2024-04-12T09:03:31.467547+0000","last_fullsized":"2024-04-12T09:03:31.467547+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:44.298234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":128,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.475801+0000","last_change":"2024-04-12T09:03:32.475801+0000","last_active":"2024-04-12T09:03:32.475801+0000","last_peered":"2024-04-12T09:03:32.475801+0000","last_clean":"2024-04-12T09:03:32.475801+0000","last_became_active":"2024-04-12T09:03:32.475490+0000","last_became_peered":"2024-04-12T09:03:32.475490+0000","last_unstale":"2024-04-12T09:03:32.475801+0000","last_undegraded":"2024-04-12T09:03:32.475801+0000","last_fullsized":"2024-04-12T09:03:32.475801+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:07:08.146234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":92,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479411+0000","last_change":"2024-04-12T09:03:32.479411+0000","last_active":"2024-04-12T09:03:32.479411+0000","last_peered":"2024-04-12T09:03:32.479411+0000","last_clean":"2024-04-12T09:03:32.479411+0000","last_became_active":"2024-04-12T09:03:32.478995+0000","last_became_peered":"2024-04-12T09:03:32.478995+0000","last_unstale":"2024-04-12T09:03:32.479411+0000","last_undegraded":"2024-04-12T09:03:32.479411+0000","last_fullsized":"2024-04-12T09:03:32.479411+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:14.135569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":118,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473165+0000","last_change":"2024-04-12T09:02:56.999377+0000","last_active":"2024-04-12T09:03:32.473165+0000","last_peered":"2024-04-12T09:03:32.473165+0000","last_clean":"2024-04-12T09:03:32.473165+0000","last_became_active":"2024-04-12T09:02:56.999004+0000","last_became_peered":"2024-04-12T09:02:56.999004+0000","last_unstale":"2024-04-12T09:03:32.473165+0000","last_undegraded":"2024-04-12T09:03:32.473165+0000","last_fullsized":"2024-04-12T09:03:32.473165+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:43:41.903567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":113,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473778+0000","last_change":"2024-04-12T08:58:56.772398+0000","last_active":"2024-04-12T09:03:32.473778+0000","last_peered":"2024-04-12T09:03:32.473778+0000","last_clean":"2024-04-12T09:03:32.473778+0000","last_became_active":"2024-04-12T08:58:56.771926+0000","last_became_peered":"2024-04-12T08:58:56.771926+0000","last_unstale":"2024-04-12T09:03:32.473778+0000","last_undegraded":"2024-04-12T09:03:32.473778+0000","last_fullsized":"2024-04-12T09:03:32.473778+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:59:29.002877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":103,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469493+0000","last_change":"2024-04-12T09:02:57.000385+0000","last_active":"2024-04-12T09:03:31.469493+0000","last_peered":"2024-04-12T09:03:31.469493+0000","last_clean":"2024-04-12T09:03:31.469493+0000","last_became_active":"2024-04-12T09:02:57.000052+0000","last_became_peered":"2024-04-12T09:02:57.000052+0000","last_unstale":"2024-04-12T09:03:31.469493+0000","last_undegraded":"2024-04-12T09:03:31.469493+0000","last_fullsized":"2024-04-12T09:03:31.469493+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:58:03.857415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"83'161","reported_seq":478,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.474613+0000","last_change":"2024-04-12T08:55:17.356828+0000","last_active":"2024-04-12T09:03:31.474613+0000","last_peered":"2024-04-12T09:03:31.474613+0000","last_clean":"2024-04-12T09:03:31.474613+0000","last_became_active":"2024-04-12T08:55:17.356374+0000","last_became_peered":"2024-04-12T08:55:17.356374+0000","last_unstale":"2024-04-12T09:03:31.474613+0000","last_undegraded":"2024-04-12T09:03:31.474613+0000","last_fullsized":"2024-04-12T09:03:31.474613+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:39:39.156733+0000","last_clean_scrub_stamp":"2024-04-12T08:39:39.156733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:05:48.388222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.480919+0000","last_change":"2024-04-12T09:03:32.480919+0000","last_active":"2024-04-12T09:03:32.480919+0000","last_peered":"2024-04-12T09:03:32.480919+0000","last_clean":"2024-04-12T09:03:32.480919+0000","last_became_active":"2024-04-12T09:03:32.480457+0000","last_became_peered":"2024-04-12T09:03:32.480457+0000","last_unstale":"2024-04-12T09:03:32.480919+0000","last_undegraded":"2024-04-12T09:03:32.480919+0000","last_fullsized":"2024-04-12T09:03:32.480919+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:15.500566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.471350+0000","last_change":"2024-04-12T08:43:29.892386+0000","last_active":"2024-04-12T09:03:31.471350+0000","last_peered":"2024-04-12T09:03:31.471350+0000","last_clean":"2024-04-12T09:03:31.471350+0000","last_became_active":"2024-04-12T08:43:29.892091+0000","last_became_peered":"2024-04-12T08:43:29.892091+0000","last_unstale":"2024-04-12T09:03:31.471350+0000","last_undegraded":"2024-04-12T09:03:31.471350+0000","last_fullsized":"2024-04-12T09:03:31.471350+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:32:11.122269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":119,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.474606+0000","last_change":"2024-04-12T08:58:56.736406+0000","last_active":"2024-04-12T09:03:31.474606+0000","last_peered":"2024-04-12T09:03:31.474606+0000","last_clean":"2024-04-12T09:03:31.474606+0000","last_became_active":"2024-04-12T08:58:56.735971+0000","last_became_peered":"2024-04-12T08:58:56.735971+0000","last_unstale":"2024-04-12T09:03:31.474606+0000","last_undegraded":"2024-04-12T09:03:31.474606+0000","last_fullsized":"2024-04-12T09:03:31.474606+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:46:54.628137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":120,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.486108+0000","last_change":"2024-04-12T09:03:32.485565+0000","last_active":"2024-04-12T09:03:32.486108+0000","last_peered":"2024-04-12T09:03:32.486108+0000","last_clean":"2024-04-12T09:03:32.486108+0000","last_became_active":"2024-04-12T09:03:32.485263+0000","last_became_peered":"2024-04-12T09:03:32.485263+0000","last_unstale":"2024-04-12T09:03:32.486108+0000","last_undegraded":"2024-04-12T09:03:32.486108+0000","last_fullsized":"2024-04-12T09:03:32.486108+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:07.310701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":130,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.478028+0000","last_change":"2024-04-12T09:02:57.003398+0000","last_active":"2024-04-12T09:03:32.478028+0000","last_peered":"2024-04-12T09:03:32.478028+0000","last_clean":"2024-04-12T09:03:32.478028+0000","last_became_active":"2024-04-12T09:02:57.003081+0000","last_became_peered":"2024-04-12T09:02:57.003081+0000","last_unstale":"2024-04-12T09:03:32.478028+0000","last_undegraded":"2024-04-12T09:03:32.478028+0000","last_fullsized":"2024-04-12T09:03:32.478028+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:50:39.887482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":102,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.475007+0000","last_change":"2024-04-12T08:55:17.424843+0000","last_active":"2024-04-12T09:03:31.475007+0000","last_peered":"2024-04-12T09:03:31.475007+0000","last_clean":"2024-04-12T09:03:31.475007+0000","last_became_active":"2024-04-12T08:55:17.424447+0000","last_became_peered":"2024-04-12T08:55:17.424447+0000","last_unstale":"2024-04-12T09:03:31.475007+0000","last_undegraded":"2024-04-12T09:03:31.475007+0000","last_fullsized":"2024-04-12T09:03:31.475007+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:59:20.792914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":118,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.474047+0000","last_change":"2024-04-12T09:02:57.005550+0000","last_active":"2024-04-12T09:03:32.474047+0000","last_peered":"2024-04-12T09:03:32.474047+0000","last_clean":"2024-04-12T09:03:32.474047+0000","last_became_active":"2024-04-12T09:02:57.005256+0000","last_became_peered":"2024-04-12T09:02:57.005256+0000","last_unstale":"2024-04-12T09:03:32.474047+0000","last_undegraded":"2024-04-12T09:03:32.474047+0000","last_fullsized":"2024-04-12T09:03:32.474047+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:27:20.156392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":112,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.476659+0000","last_change":"2024-04-12T08:55:17.359973+0000","last_active":"2024-04-12T09:03:32.476659+0000","last_peered":"2024-04-12T09:03:32.476659+0000","last_clean":"2024-04-12T09:03:32.476659+0000","last_became_active":"2024-04-12T08:55:17.357844+0000","last_became_peered":"2024-04-12T08:55:17.357844+0000","last_unstale":"2024-04-12T09:03:32.476659+0000","last_undegraded":"2024-04-12T09:03:32.476659+0000","last_fullsized":"2024-04-12T09:03:32.476659+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:57:50.740134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":71,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.484186+0000","last_change":"2024-04-12T09:03:32.484186+0000","last_active":"2024-04-12T09:03:32.484186+0000","last_peered":"2024-04-12T09:03:32.484186+0000","last_clean":"2024-04-12T09:03:32.484186+0000","last_became_active":"2024-04-12T09:03:32.482981+0000","last_became_peered":"2024-04-12T09:03:32.482981+0000","last_unstale":"2024-04-12T09:03:32.484186+0000","last_undegraded":"2024-04-12T09:03:32.484186+0000","last_fullsized":"2024-04-12T09:03:32.484186+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:50:05.928185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":109,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.473656+0000","last_change":"2024-04-12T08:58:56.737968+0000","last_active":"2024-04-12T09:03:32.473656+0000","last_peered":"2024-04-12T09:03:32.473656+0000","last_clean":"2024-04-12T09:03:32.473656+0000","last_became_active":"2024-04-12T08:58:56.737579+0000","last_became_peered":"2024-04-12T08:58:56.737579+0000","last_unstale":"2024-04-12T09:03:32.473656+0000","last_undegraded":"2024-04-12T09:03:32.473656+0000","last_fullsized":"2024-04-12T09:03:32.473656+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:09:45.836715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.484123+0000","last_change":"2024-04-12T09:03:32.484123+0000","last_active":"2024-04-12T09:03:32.484123+0000","last_peered":"2024-04-12T09:03:32.484123+0000","last_clean":"2024-04-12T09:03:32.484123+0000","last_became_active":"2024-04-12T09:03:32.483052+0000","last_became_peered":"2024-04-12T09:03:32.483052+0000","last_unstale":"2024-04-12T09:03:32.484123+0000","last_undegraded":"2024-04-12T09:03:32.484123+0000","last_fullsized":"2024-04-12T09:03:32.484123+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:01:06.479551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":102,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.471135+0000","last_change":"2024-04-12T09:02:57.007543+0000","last_active":"2024-04-12T09:03:31.471135+0000","last_peered":"2024-04-12T09:03:31.471135+0000","last_clean":"2024-04-12T09:03:31.471135+0000","last_became_active":"2024-04-12T09:02:57.007234+0000","last_became_peered":"2024-04-12T09:02:57.007234+0000","last_unstale":"2024-04-12T09:03:31.471135+0000","last_undegraded":"2024-04-12T09:03:31.471135+0000","last_fullsized":"2024-04-12T09:03:31.471135+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:42:17.324060+0000","last_clean_scrub_stamp":"2024-04-12T08:42:17.324060+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:35.310595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":130,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.477688+0000","last_change":"2024-04-12T09:02:57.005094+0000","last_active":"2024-04-12T09:03:32.477688+0000","last_peered":"2024-04-12T09:03:32.477688+0000","last_clean":"2024-04-12T09:03:32.477688+0000","last_became_active":"2024-04-12T09:02:57.004691+0000","last_became_peered":"2024-04-12T09:02:57.004691+0000","last_unstale":"2024-04-12T09:03:32.477688+0000","last_undegraded":"2024-04-12T09:03:32.477688+0000","last_fullsized":"2024-04-12T09:03:32.477688+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:06:07.486319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.466794+0000","last_change":"2024-04-12T08:43:28.365741+0000","last_active":"2024-04-12T09:03:31.466794+0000","last_peered":"2024-04-12T09:03:31.466794+0000","last_clean":"2024-04-12T09:03:31.466794+0000","last_became_active":"2024-04-12T08:43:28.365425+0000","last_became_peered":"2024-04-12T08:43:28.365425+0000","last_unstale":"2024-04-12T09:03:31.466794+0000","last_undegraded":"2024-04-12T09:03:31.466794+0000","last_fullsized":"2024-04-12T09:03:31.466794+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:13:45.303498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":114,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.476716+0000","last_change":"2024-04-12T08:55:17.355386+0000","last_active":"2024-04-12T09:03:32.476716+0000","last_peered":"2024-04-12T09:03:32.476716+0000","last_clean":"2024-04-12T09:03:32.476716+0000","last_became_active":"2024-04-12T08:55:17.353579+0000","last_became_peered":"2024-04-12T08:55:17.353579+0000","last_unstale":"2024-04-12T09:03:32.476716+0000","last_undegraded":"2024-04-12T09:03:32.476716+0000","last_fullsized":"2024-04-12T09:03:32.476716+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T10:56:14.416234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":95,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.479337+0000","last_change":"2024-04-12T09:03:32.479337+0000","last_active":"2024-04-12T09:03:32.479337+0000","last_peered":"2024-04-12T09:03:32.479337+0000","last_clean":"2024-04-12T09:03:32.479337+0000","last_became_active":"2024-04-12T09:03:32.478916+0000","last_became_peered":"2024-04-12T09:03:32.478916+0000","last_unstale":"2024-04-12T09:03:32.479337+0000","last_undegraded":"2024-04-12T09:03:32.479337+0000","last_fullsized":"2024-04-12T09:03:32.479337+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:54:08.702850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":129,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469699+0000","last_change":"2024-04-12T08:58:56.736545+0000","last_active":"2024-04-12T09:03:31.469699+0000","last_peered":"2024-04-12T09:03:31.469699+0000","last_clean":"2024-04-12T09:03:31.469699+0000","last_became_active":"2024-04-12T08:58:56.736182+0000","last_became_peered":"2024-04-12T08:58:56.736182+0000","last_unstale":"2024-04-12T09:03:31.469699+0000","last_undegraded":"2024-04-12T09:03:31.469699+0000","last_fullsized":"2024-04-12T09:03:31.469699+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:39:18.004066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.484285+0000","last_change":"2024-04-12T09:03:32.484285+0000","last_active":"2024-04-12T09:03:32.484285+0000","last_peered":"2024-04-12T09:03:32.484285+0000","last_clean":"2024-04-12T09:03:32.484285+0000","last_became_active":"2024-04-12T09:03:32.483889+0000","last_became_peered":"2024-04-12T09:03:32.483889+0000","last_unstale":"2024-04-12T09:03:32.484285+0000","last_undegraded":"2024-04-12T09:03:32.484285+0000","last_fullsized":"2024-04-12T09:03:32.484285+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:13:29.920796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":84,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.470815+0000","last_change":"2024-04-12T08:43:28.137332+0000","last_active":"2024-04-12T09:03:31.470815+0000","last_peered":"2024-04-12T09:03:31.470815+0000","last_clean":"2024-04-12T09:03:31.470815+0000","last_became_active":"2024-04-12T08:43:28.136749+0000","last_became_peered":"2024-04-12T08:43:28.136749+0000","last_unstale":"2024-04-12T09:03:31.470815+0000","last_undegraded":"2024-04-12T09:03:31.470815+0000","last_fullsized":"2024-04-12T09:03:31.470815+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:20:09.223627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.483824+0000","last_change":"2024-04-12T09:03:32.483824+0000","last_active":"2024-04-12T09:03:32.483824+0000","last_peered":"2024-04-12T09:03:32.483824+0000","last_clean":"2024-04-12T09:03:32.483824+0000","last_became_active":"2024-04-12T09:03:32.483434+0000","last_became_peered":"2024-04-12T09:03:32.483434+0000","last_unstale":"2024-04-12T09:03:32.483824+0000","last_undegraded":"2024-04-12T09:03:32.483824+0000","last_fullsized":"2024-04-12T09:03:32.483824+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:13:33.249176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":107,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.481382+0000","last_change":"2024-04-12T09:03:32.481382+0000","last_active":"2024-04-12T09:03:32.481382+0000","last_peered":"2024-04-12T09:03:32.481382+0000","last_clean":"2024-04-12T09:03:32.481382+0000","last_became_active":"2024-04-12T09:03:32.480957+0000","last_became_peered":"2024-04-12T09:03:32.480957+0000","last_unstale":"2024-04-12T09:03:32.481382+0000","last_undegraded":"2024-04-12T09:03:32.481382+0000","last_fullsized":"2024-04-12T09:03:32.481382+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:06:39.854442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":88,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.469500+0000","last_change":"2024-04-12T09:02:57.004852+0000","last_active":"2024-04-12T09:03:31.469500+0000","last_peered":"2024-04-12T09:03:31.469500+0000","last_clean":"2024-04-12T09:03:31.469500+0000","last_became_active":"2024-04-12T09:02:57.004532+0000","last_became_peered":"2024-04-12T09:02:57.004532+0000","last_unstale":"2024-04-12T09:03:31.469500+0000","last_undegraded":"2024-04-12T09:03:31.469500+0000","last_fullsized":"2024-04-12T09:03:31.469500+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T09:08:22.187536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":124,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-12T09:03:32.476047+0000","last_change":"2024-04-12T09:02:57.005618+0000","last_active":"2024-04-12T09:03:32.476047+0000","last_peered":"2024-04-12T09:03:32.476047+0000","last_clean":"2024-04-12T09:03:32.476047+0000","last_became_active":"2024-04-12T09:02:57.005238+0000","last_became_peered":"2024-04-12T09:02:57.005238+0000","last_unstale":"2024-04-12T09:03:32.476047+0000","last_undegraded":"2024-04-12T09:03:32.476047+0000","last_fullsized":"2024-04-12T09:03:32.476047+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:27.102515+0000","last_clean_scrub_stamp":"2024-04-12T08:43:27.102515+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:23:01.772101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-12T09:03:31.466794+0000","last_change":"2024-04-12T09:02:57.000721+0000","last_active":"2024-04-12T09:03:31.466794+0000","last_peered":"2024-04-12T09:03:31.466794+0000","last_clean":"2024-04-12T09:03:31.466794+0000","last_became_active":"2024-04-12T09:02:57.000236+0000","last_became_peered":"2024-04-12T09:02:57.000236+0000","last_unstale":"2024-04-12T09:03:31.466794+0000","last_undegraded":"2024-04-12T09:03:31.466794+0000","last_fullsized":"2024-04-12T09:03:31.466794+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T08:43:28.109238+0000","last_clean_scrub_stamp":"2024-04-12T08:43:28.109238+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:00:46.597355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4834,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15383,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201558,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299652,"kb_used_data":2288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445500,"statfs":{"total":95995035648,"available":95688192000,"internally_reserved":0,"allocated":2342912,"data_stored":1501158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463194,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303724,"kb_used_data":2272,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93441428,"statfs":{"total":95995035648,"available":95684022272,"internally_reserved":0,"allocated":2326528,"data_stored":1500564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":116,"seq":498216206405,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":181992,"kb_used_data":1684,"kb_used_omap":5,"kb_used_meta":180282,"kb_avail":93563160,"statfs":{"total":95995035648,"available":95808675840,"internally_reserved":0,"allocated":1724416,"data_stored":905823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5934,"internal_metadata":184608978},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":95,"seq":408021893233,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":262404,"kb_used_data":2288,"kb_used_omap":5,"kb_used_meta":260090,"kb_avail":93482748,"statfs":{"total":95995035648,"available":95726333952,"internally_reserved":0,"allocated":2342912,"data_stored":1501776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6054,"internal_metadata":266332250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":133,"seq":571230650389,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":61748,"kb_used_data":1696,"kb_used_omap":6,"kb_used_meta":60025,"kb_avail":93683404,"statfs":{"total":95995035648,"available":95931805696,"internally_reserved":0,"allocated":1736704,"data_stored":906851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6732,"internal_metadata":61466036},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":138,"seq":592705486862,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":80488,"kb_used_data":1692,"kb_used_omap":4,"kb_used_meta":78779,"kb_avail":93664664,"statfs":{"total":95995035648,"available":95912615936,"internally_reserved":0,"allocated":1732608,"data_stored":906337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4638,"internal_metadata":80670178},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986468,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599732,"kb_used_data":1696,"kb_used_omap":1,"kb_used_meta":598014,"kb_avail":93145420,"statfs":{"total":95995035648,"available":95380910080,"internally_reserved":0,"allocated":1736704,"data_stored":906283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612366680},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724831,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907308,"kb_used_data":1696,"kb_used_omap":1,"kb_used_meta":905598,"kb_avail":92837844,"statfs":{"total":95995035648,"available":95065952256,"internally_reserved":0,"allocated":1736704,"data_stored":906875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":927332696},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4350,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4230,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2934,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T09:04:35.894 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-12T09:04:35.894 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-12T09:04:35.894 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-12T09:04:35.894 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:04:36.154 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:36.537 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:36 smithi090 ceph-mon[99575]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T09:04:36.537 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:36 smithi090 ceph-mon[99575]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:36.537 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:36 smithi090 ceph-mon[95814]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T09:04:36.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:36 smithi090 ceph-mon[95814]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:36 smithi114 ceph-mon[89482]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T09:04:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:36 smithi114 ceph-mon[89482]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:38.054 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:04:38.054 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:04:38.746 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:38 smithi090 ceph-mon[95814]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:38.747 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:38 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3799964557' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:38.747 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:38 smithi090 ceph-mon[99575]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:38.747 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:38 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3799964557' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:38.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:38 smithi114 ceph-mon[89482]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:38.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:38 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3799964557' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:40.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:40 smithi114 ceph-mon[89482]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:40 smithi090 ceph-mon[99575]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:40 smithi090 ceph-mon[95814]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:41.774 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:04:42.045 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:42.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:42 smithi114 ceph-mon[89482]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:42 smithi090 ceph-mon[99575]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:42.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:42 smithi090 ceph-mon[95814]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:43.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:04:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:04:43] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.43.0" 2024-04-12T09:04:43.932 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:04:43.932 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:04:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:44 smithi114 ceph-mon[89482]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:44 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/54476797' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:44 smithi090 ceph-mon[99575]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:44 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/54476797' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:44 smithi090 ceph-mon[95814]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:44 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/54476797' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:45.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:45.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:04:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:46 smithi114 ceph-mon[89482]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:46.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:46 smithi090 ceph-mon[99575]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:46 smithi090 ceph-mon[95814]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:47.594 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:04:47.859 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:47.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:47 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:04:47.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:47 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:04:47.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:47 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:04:48.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:48 smithi114 ceph-mon[89482]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:48.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:48 smithi090 ceph-mon[99575]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:48 smithi090 ceph-mon[95814]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:49.782 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:04:49.782 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:04:50.862 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:50 smithi090 ceph-mon[95814]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:50.862 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:50 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1245990608' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:50.863 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:50 smithi090 ceph-mon[99575]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:50.863 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:50 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1245990608' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:50 smithi114 ceph-mon[89482]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:50 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1245990608' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:52.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:52 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:52.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:52 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:52.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:52 smithi090 ceph-mon[95814]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:52.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:52 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:52.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:52 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:52.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:52 smithi090 ceph-mon[99575]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:52.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:52 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:52.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:52 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:52.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:52 smithi114 ceph-mon[89482]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:53.452 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:04:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:04:53] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.43.0" 2024-04-12T09:04:53.888 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:04:54.160 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:04:54.592 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:54.592 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:54.592 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[99575]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:54.592 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:04:54.593 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:04:54.593 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:54.593 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:54.593 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:54.593 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[95814]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:54.593 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:04:54.593 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:04:54.593 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:54 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:54.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:54 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:54.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:54 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:54.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:54 smithi114 ceph-mon[89482]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:54.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:54 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:04:54.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:54 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:04:54.618 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:54 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:04:55.512 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:55 smithi090 ceph-mon[99575]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:55.512 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:55 smithi090 ceph-mon[99575]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:55.512 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:55 smithi090 ceph-mon[95814]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:55.512 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:55 smithi090 ceph-mon[95814]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:55.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:55 smithi114 ceph-mon[89482]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:55.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:55 smithi114 ceph-mon[89482]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:56.110 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:04:56.111 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:04:56.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:56 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1549341436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:56.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:56 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1549341436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:56.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:56 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1549341436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:04:57.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:57 smithi114 ceph-mon[89482]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:57.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:57 smithi090 ceph-mon[99575]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:57.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:57 smithi090 ceph-mon[95814]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:59.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:04:59 smithi114 ceph-mon[89482]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:59.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:04:59 smithi090 ceph-mon[99575]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:59.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:04:59 smithi090 ceph-mon[95814]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:04:59.785 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:00.057 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:01.493 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:01 smithi090 ceph-mon[99575]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:01.493 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:01.493 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:01 smithi090 ceph-mon[95814]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:01.493 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:01.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:01 smithi114 ceph-mon[89482]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:01.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:01 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:01.962 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:05:01.962 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:05:02.548 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:02 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1021442220' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:02.548 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:02 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1021442220' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:02.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:02 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1021442220' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:03.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:03 smithi114 ceph-mon[89482]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:03.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:03 smithi090 ceph-mon[95814]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:03.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:05:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:05:03] "GET /metrics HTTP/1.1" 200 34172 "" "Prometheus/2.43.0" 2024-04-12T09:05:03.674 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:03 smithi090 ceph-mon[99575]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:05.574 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:05.605 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:05 smithi090 ceph-mon[99575]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:05.605 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:05 smithi090 ceph-mon[95814]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:05.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:05 smithi114 ceph-mon[89482]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:05.841 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:07.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:07 smithi114 ceph-mon[89482]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:07.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:07 smithi090 ceph-mon[95814]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:07.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:07 smithi090 ceph-mon[99575]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:05:07.718 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:05:08.421 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:08 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3367395996' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:08.421 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:08 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3367395996' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:08.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:08 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3367395996' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:09.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:09 smithi114 ceph-mon[89482]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:09.645 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:09 smithi090 ceph-mon[95814]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:09.646 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:09 smithi090 ceph-mon[99575]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:11.448 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:11.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:11 smithi114 ceph-mon[89482]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:11.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:11 smithi090 ceph-mon[99575]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:11.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:11 smithi090 ceph-mon[95814]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:11.720 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:13.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:13 smithi090 ceph-mon[99575]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:13.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:13 smithi090 ceph-mon[95814]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:13.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:05:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:05:13] "GET /metrics HTTP/1.1" 200 34172 "" "Prometheus/2.43.0" 2024-04-12T09:05:13.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:13 smithi114 ceph-mon[89482]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:13.624 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:05:13.625 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:05:14.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:14 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4025571263' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:14.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:14 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4025571263' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:14.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:14 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4025571263' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:15.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:15 smithi114 ceph-mon[89482]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:15.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:15 smithi090 ceph-mon[99575]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:15.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:15 smithi090 ceph-mon[95814]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:16.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:16 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:16.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:16 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:16.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:16 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:17.280 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:17.548 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:17.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:17 smithi114 ceph-mon[89482]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:17.670 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:17 smithi090 ceph-mon[99575]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:17.670 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:17 smithi090 ceph-mon[95814]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:19.399 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:05:19.399 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:05:19.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:19 smithi114 ceph-mon[89482]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:19.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:19 smithi090 ceph-mon[99575]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:19.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:19 smithi090 ceph-mon[95814]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:20.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:20 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3137086824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:20.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:20 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3137086824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:20.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:20 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3137086824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:21.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:21 smithi114 ceph-mon[89482]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:21.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:21 smithi090 ceph-mon[95814]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:21.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:21 smithi090 ceph-mon[99575]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:23.114 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:23.384 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:23.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:23 smithi090 ceph-mon[95814]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:23.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:23 smithi090 ceph-mon[99575]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:23.424 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:05:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:05:23] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.43.0" 2024-04-12T09:05:23.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:23 smithi114 ceph-mon[89482]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:25.309 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:05:25.310 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:05:25.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:25 smithi114 ceph-mon[89482]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:25.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1482011936' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:25.618 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:25 smithi090 ceph-mon[99575]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:25.618 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1482011936' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:25.618 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:25 smithi090 ceph-mon[95814]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:25.618 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1482011936' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:27.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:27 smithi114 ceph-mon[89482]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:27.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:27 smithi090 ceph-mon[95814]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:27.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:27 smithi090 ceph-mon[99575]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:28.931 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:29.201 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:29.577 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:29 smithi090 ceph-mon[99575]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:29.577 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:29 smithi090 ceph-mon[95814]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:29.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:29 smithi114 ceph-mon[89482]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:31.111 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:05:31.112 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:05:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:31 smithi114 ceph-mon[89482]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:31.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:31 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4177080551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:31 smithi090 ceph-mon[99575]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:31.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:31 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4177080551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:31 smithi090 ceph-mon[95814]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:31.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:31 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4177080551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:33.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:33 smithi114 ceph-mon[89482]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:33.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:33 smithi090 ceph-mon[99575]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:33.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:33 smithi090 ceph-mon[95814]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:05:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:05:33] "GET /metrics HTTP/1.1" 200 34169 "" "Prometheus/2.43.0" 2024-04-12T09:05:34.846 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:35.134 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:35.580 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:35 smithi090 ceph-mon[95814]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:35.581 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:35 smithi090 ceph-mon[99575]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:35.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:35 smithi114 ceph-mon[89482]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:36.978 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:05:36.978 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:05:37.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:37 smithi114 ceph-mon[89482]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:37.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:37 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1768596088' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:37.629 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:37 smithi090 ceph-mon[99575]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:37.629 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:37 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1768596088' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:37.629 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:37 smithi090 ceph-mon[95814]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:37.629 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:37 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1768596088' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:39.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:39 smithi114 ceph-mon[89482]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:39.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:39 smithi090 ceph-mon[95814]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:39.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:39 smithi090 ceph-mon[99575]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:40.656 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:40.924 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:41.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:41 smithi114 ceph-mon[89482]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:41.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:41 smithi090 ceph-mon[95814]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:41.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:41 smithi090 ceph-mon[99575]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:43.038 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:05:43.038 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:05:43.320 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:05:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:05:43] "GET /metrics HTTP/1.1" 200 34169 "" "Prometheus/2.43.0" 2024-04-12T09:05:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:43 smithi114 ceph-mon[89482]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:43.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:43 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1653384580' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:43.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:43 smithi090 ceph-mon[99575]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:43.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:43 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1653384580' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:43.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:43 smithi090 ceph-mon[95814]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:43.674 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:43 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1653384580' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:45.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:45 smithi114 ceph-mon[89482]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:45.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:45 smithi090 ceph-mon[99575]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:45.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:45 smithi090 ceph-mon[95814]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:46.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:46 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:46.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:46 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:46.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:46 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:05:46.711 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:46.980 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:47.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:47 smithi114 ceph-mon[89482]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:47.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:47 smithi090 ceph-mon[99575]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:47.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:47 smithi090 ceph-mon[95814]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:49.122 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:05:49.122 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:05:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:49 smithi090 ceph-mon[99575]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:49.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:49 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2258445670' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:49.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:49 smithi090 ceph-mon[95814]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:49.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:49 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2258445670' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:49.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:49 smithi114 ceph-mon[89482]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:49.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:49 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2258445670' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:51.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:51 smithi114 ceph-mon[89482]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:51.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:51 smithi090 ceph-mon[99575]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:51.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:51 smithi090 ceph-mon[95814]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:52.859 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:53.128 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:53.508 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:53 smithi090 ceph-mon[95814]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:53.508 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:05:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:05:53] "GET /metrics HTTP/1.1" 200 34174 "" "Prometheus/2.43.0" 2024-04-12T09:05:53.508 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:53 smithi090 ceph-mon[99575]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:53.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:53 smithi114 ceph-mon[89482]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:54.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:54 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:05:54.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:54 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:05:54.721 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:54 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:05:55.199 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:05:55.199 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:05:55.561 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:55 smithi090 ceph-mon[99575]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:55.561 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:55 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3169644850' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:55.561 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:55 smithi090 ceph-mon[95814]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:55.561 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:55 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3169644850' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:55.675 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:55 smithi114 ceph-mon[89482]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:55.675 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:55 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3169644850' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:05:57.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:57 smithi090 ceph-mon[99575]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:57.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:57 smithi090 ceph-mon[95814]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:57.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:57 smithi114 ceph-mon[89482]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:58 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:58 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:58 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:58 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:58 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:58 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.300 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:05:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:05:59.937 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[99575]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:59.937 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.937 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.937 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:05:59.937 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:05:59.937 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.938 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[95814]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:05:59.938 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.938 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:05:59.938 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:05:59.938 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:05:59.938 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:05:59 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:06:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:59 smithi114 ceph-mon[89482]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:59 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:06:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:59 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:06:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:59 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:06:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:59 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:06:00.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:05:59 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:06:01.039 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:00 smithi090 ceph-mon[99575]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:01.039 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:00 smithi090 ceph-mon[99575]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:01.039 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:01.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:00 smithi090 ceph-mon[95814]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:01.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:00 smithi090 ceph-mon[95814]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:01.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:00 smithi114 ceph-mon[89482]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:00 smithi114 ceph-mon[89482]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:01.504 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:06:01.504 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:06:01.829 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:01 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1898605789' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:01.829 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:01 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1898605789' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:02.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:01 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1898605789' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:02 smithi114 ceph-mon[89482]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:03.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:02 smithi090 ceph-mon[95814]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:03.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:02 smithi090 ceph-mon[99575]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:03.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:06:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:06:03] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2024-04-12T09:06:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:04 smithi114 ceph-mon[89482]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:05.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:04 smithi090 ceph-mon[99575]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:04 smithi090 ceph-mon[95814]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:05.184 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:06:05.451 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:06:06.907 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:06 smithi090 ceph-mon[95814]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:06.907 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:06 smithi090 ceph-mon[99575]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:07.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:06 smithi114 ceph-mon[89482]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:07.415 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:06:07.416 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:06:08.114 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:07 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3001578792' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:08.114 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:07 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3001578792' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:08.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:07 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3001578792' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:09.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:08 smithi114 ceph-mon[89482]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:09.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:08 smithi090 ceph-mon[99575]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:09.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:08 smithi090 ceph-mon[95814]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:11.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:10 smithi114 ceph-mon[89482]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:11.141 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:06:11.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:10 smithi090 ceph-mon[99575]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:11.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:10 smithi090 ceph-mon[95814]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:11.409 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:06:12.892 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:12 smithi090 ceph-mon[95814]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:12.892 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:12 smithi090 ceph-mon[99575]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:13.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:12 smithi114 ceph-mon[89482]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:13.390 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:06:13.390 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:06:13.436 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:06:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:06:13] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2024-04-12T09:06:14.055 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:13 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1091219618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:14.055 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:13 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1091219618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:14.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:13 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1091219618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:15.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:14 smithi114 ceph-mon[89482]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:15.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:14 smithi090 ceph-mon[95814]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:15.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:14 smithi090 ceph-mon[99575]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:16.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:15 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:16.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:15 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:16.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:17.083 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:06:17.114 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:16 smithi090 ceph-mon[99575]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:17.115 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:16 smithi090 ceph-mon[95814]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:16 smithi114 ceph-mon[89482]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:17.353 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:06:18.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:17 smithi114 ceph-mon[89482]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:18.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:17 smithi090 ceph-mon[99575]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:18.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:17 smithi090 ceph-mon[95814]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:19.323 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:06:19.323 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:06:19.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:19 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/116371501' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:19.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:19 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/116371501' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:19.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:19 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/116371501' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:20.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:20 smithi090 ceph-mon[99575]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:20.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:20 smithi090 ceph-mon[95814]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:20 smithi114 ceph-mon[89482]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:22.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:22 smithi090 ceph-mon[99575]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:22.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:22 smithi090 ceph-mon[95814]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:22.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:22 smithi114 ceph-mon[89482]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:23.037 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:06:23.307 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:06:23.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:06:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:06:23] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.43.0" 2024-04-12T09:06:24.638 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:24 smithi090 ceph-mon[99575]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:24.639 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:24 smithi090 ceph-mon[95814]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:24.669 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:24 smithi114 ceph-mon[89482]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:25.236 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:06:25.236 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:06:25.622 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1085586504' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:25.622 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1085586504' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:25.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1085586504' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:26.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:26 smithi090 ceph-mon[95814]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:26.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:26 smithi090 ceph-mon[99575]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:26 smithi114 ceph-mon[89482]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:28.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:28 smithi090 ceph-mon[99575]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:28.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:28 smithi090 ceph-mon[95814]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:28 smithi114 ceph-mon[89482]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:28.935 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:06:29.204 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:06:30.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:30 smithi090 ceph-mon[99575]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:30.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:30 smithi090 ceph-mon[95814]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:30.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:30 smithi114 ceph-mon[89482]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:31.164 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:06:31.164 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:06:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:31.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:31 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2011444689' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:31.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:31 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2011444689' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:31 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2011444689' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:32.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:32 smithi090 ceph-mon[95814]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:32.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:32 smithi090 ceph-mon[99575]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:32.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:32 smithi114 ceph-mon[89482]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:33.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:06:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:06:33] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.43.0" 2024-04-12T09:06:34.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:34 smithi090 ceph-mon[95814]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:34.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:34 smithi090 ceph-mon[99575]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:34.799 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:06:34.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:34 smithi114 ceph-mon[89482]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:35.069 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:06:36.526 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:36 smithi090 ceph-mon[99575]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:36.526 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:36 smithi090 ceph-mon[95814]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:36.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:36 smithi114 ceph-mon[89482]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:37.013 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:06:37.014 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:06:37.607 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:37 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3078868194' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:37.607 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:37 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3078868194' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:37.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:37 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3078868194' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:38.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:38 smithi090 ceph-mon[99575]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:38.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:38 smithi090 ceph-mon[95814]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:38.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:38 smithi114 ceph-mon[89482]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:40.632 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:06:40.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:40 smithi114 ceph-mon[89482]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:40.902 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:06:40.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:40 smithi090 ceph-mon[95814]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:40 smithi090 ceph-mon[99575]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:42.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:42 smithi090 ceph-mon[99575]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:42.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:42 smithi090 ceph-mon[95814]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:42.836 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:06:42.836 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:06:42.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:42 smithi114 ceph-mon[89482]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:43.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:06:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:06:43] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.43.0" 2024-04-12T09:06:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:43 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/378614522' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:43 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/378614522' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:43.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:43 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/378614522' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:44.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:44 smithi114 ceph-mon[89482]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:44 smithi090 ceph-mon[99575]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:44 smithi090 ceph-mon[95814]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:46.498 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:06:46.767 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:06:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:46 smithi114 ceph-mon[89482]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:46.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:46 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:46.889 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:46 smithi090 ceph-mon[99575]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:46.889 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:46 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:46.889 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:46 smithi090 ceph-mon[95814]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:46.889 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:46 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:06:48.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:48 smithi090 ceph-mon[99575]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:48.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:48 smithi090 ceph-mon[95814]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:48.685 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:06:48.685 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:06:48.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:48 smithi114 ceph-mon[89482]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:49.580 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:49 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1299118110' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:49.580 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:49 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1299118110' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:49.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:49 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1299118110' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:50.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:50 smithi114 ceph-mon[89482]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:50.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:50 smithi090 ceph-mon[99575]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:50 smithi090 ceph-mon[95814]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:52.318 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:06:52.593 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:06:52.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:52 smithi090 ceph-mon[99575]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:52.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:52 smithi090 ceph-mon[95814]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:52.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:52 smithi114 ceph-mon[89482]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:06:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:06:53] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.43.0" 2024-04-12T09:06:54.773 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:06:54.773 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:06:54.774 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:54 smithi090 ceph-mon[95814]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:54.774 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:54 smithi090 ceph-mon[99575]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:54.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:54 smithi114 ceph-mon[89482]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:55.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:55 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/226529161' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:55 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/226529161' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:55.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:55 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/226529161' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:06:56.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:56 smithi114 ceph-mon[89482]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:56 smithi090 ceph-mon[95814]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:56 smithi090 ceph-mon[99575]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:58.486 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:06:58.756 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:06:58.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:58 smithi114 ceph-mon[89482]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:58.875 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:58 smithi090 ceph-mon[99575]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:58.875 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:58 smithi090 ceph-mon[95814]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:06:59.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:06:59 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:06:59.878 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:06:59 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:06:59.879 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:06:59 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:07:00.866 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:07:00.867 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:07:00.867 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:00 smithi090 ceph-mon[99575]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:00.867 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:00 smithi090 ceph-mon[95814]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:00.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:00 smithi114 ceph-mon[89482]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:01.722 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:01 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:01.722 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:01 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3657854034' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:01.722 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:01 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:01.723 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:01 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3657854034' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:01.746 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:01 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:01.746 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:01 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3657854034' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:02.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:02 smithi114 ceph-mon[89482]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:02.741 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:02 smithi090 ceph-mon[99575]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:02.741 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:02 smithi090 ceph-mon[95814]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:03.566 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:07:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:07:03] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.43.0" 2024-04-12T09:07:04.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:04 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:04.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:04 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:04.367 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:04 smithi114 ceph-mon[89482]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:04.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:04 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:04.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:04 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:04.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:04 smithi090 ceph-mon[99575]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:04 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:04 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:04.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:04 smithi090 ceph-mon[95814]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:04.752 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:07:05.017 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:07:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:05 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:05 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:05 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:07:05.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:05 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:07:05.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:05 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:05 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:05 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:05 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:07:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:05 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:07:05.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:05 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:05.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:05 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:05.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:05 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:05.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:05 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:07:05.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:05 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:07:05.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:05 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:07:06.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:06 smithi090 ceph-mon[95814]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:06.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:06 smithi090 ceph-mon[95814]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:06.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:06 smithi090 ceph-mon[99575]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:06.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:06 smithi090 ceph-mon[99575]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:06 smithi114 ceph-mon[89482]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:06.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:06 smithi114 ceph-mon[89482]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:06.883 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:07:06.883 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:07:07.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:07 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2846594175' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:07.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:07 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2846594175' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:07.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:07 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2846594175' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:08.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:08 smithi114 ceph-mon[89482]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:08.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:08 smithi090 ceph-mon[99575]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:08.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:08 smithi090 ceph-mon[95814]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:10.573 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:07:10.849 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:07:10.868 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:10 smithi114 ceph-mon[89482]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:10.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:10 smithi090 ceph-mon[99575]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:10.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:10 smithi090 ceph-mon[95814]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:12.757 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:07:12.758 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:07:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:12 smithi114 ceph-mon[89482]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:12.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:12 smithi090 ceph-mon[99575]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:12 smithi090 ceph-mon[95814]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:13.418 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:07:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:07:13] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.43.0" 2024-04-12T09:07:13.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:13 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3283029445' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:13.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:13 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3283029445' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:13 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3283029445' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:14.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:14 smithi114 ceph-mon[89482]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:14 smithi090 ceph-mon[95814]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:14.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:14 smithi090 ceph-mon[99575]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:15 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:15 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:16.444 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:07:16.714 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:07:16.837 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:16 smithi090 ceph-mon[99575]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:16.837 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:16 smithi090 ceph-mon[95814]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:16.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:16 smithi114 ceph-mon[89482]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:18.600 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:07:18.600 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:07:18.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:18 smithi114 ceph-mon[89482]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:18.920 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:18 smithi090 ceph-mon[95814]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:18.920 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:18 smithi090 ceph-mon[99575]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:19.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:19 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1124396391' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:19.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:19 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1124396391' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:19.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:19 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1124396391' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:20.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:20 smithi114 ceph-mon[89482]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:20.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:20 smithi090 ceph-mon[95814]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:20.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:20 smithi090 ceph-mon[99575]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:22.256 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:07:22.528 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:07:22.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:22 smithi114 ceph-mon[89482]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:22.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:22 smithi090 ceph-mon[99575]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:22.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:22 smithi090 ceph-mon[95814]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:23.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:07:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:07:23] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2024-04-12T09:07:24.560 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:07:24.560 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:07:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:24 smithi114 ceph-mon[89482]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:24.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:24 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1750021352' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:24 smithi090 ceph-mon[99575]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:24.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:24 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1750021352' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:24.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:24 smithi090 ceph-mon[95814]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:24.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:24 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1750021352' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:26.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:26 smithi114 ceph-mon[89482]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:26.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:26 smithi090 ceph-mon[95814]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:26.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:26 smithi090 ceph-mon[99575]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:28.271 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:07:28.540 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:07:28.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:28 smithi114 ceph-mon[89482]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:28.906 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:28 smithi090 ceph-mon[99575]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:28.907 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:28 smithi090 ceph-mon[95814]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:30.462 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:07:30.462 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:07:30.859 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:30 smithi090 ceph-mon[99575]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:30.859 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:30 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/3330696489' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:30.859 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:30 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:30.859 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:30 smithi090 ceph-mon[95814]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:30.859 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:30 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/3330696489' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:30.859 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:30 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:30 smithi114 ceph-mon[89482]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:30 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/3330696489' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:30 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:32.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:32 smithi090 ceph-mon[99575]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:32.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:32 smithi090 ceph-mon[95814]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:33.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:32 smithi114 ceph-mon[89482]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:07:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:07:33] "GET /metrics HTTP/1.1" 200 34172 "" "Prometheus/2.43.0" 2024-04-12T09:07:34.177 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:07:34.432 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:07:34.862 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:34 smithi090 ceph-mon[99575]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:34.863 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:34 smithi090 ceph-mon[95814]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:35.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:34 smithi114 ceph-mon[89482]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:36.541 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:07:36.541 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:07:36.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:36 smithi090 ceph-mon[99575]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:36.912 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:36 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/398358266' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:36.912 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:36 smithi090 ceph-mon[95814]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:36.913 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:36 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/398358266' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:36.944 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:36 smithi114 ceph-mon[89482]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:36.944 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:36 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/398358266' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:38.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:38 smithi090 ceph-mon[99575]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:38.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:38 smithi090 ceph-mon[95814]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:39.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:38 smithi114 ceph-mon[89482]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:40.388 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:07:40.659 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:07:40.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:40 smithi090 ceph-mon[99575]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:40.672 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:40 smithi090 ceph-mon[95814]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:41.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:40 smithi114 ceph-mon[89482]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:42.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:42 smithi090 ceph-mon[95814]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:42.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:42 smithi090 ceph-mon[99575]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:42.859 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:07:42.859 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:07:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:42 smithi114 ceph-mon[89482]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:43.636 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:07:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:07:43] "GET /metrics HTTP/1.1" 200 34172 "" "Prometheus/2.43.0" 2024-04-12T09:07:43.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:43 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/150626561' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:43.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:43 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/150626561' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:44.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:43 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/150626561' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:44 smithi090 ceph-mon[99575]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:44 smithi090 ceph-mon[95814]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:45.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:44 smithi114 ceph-mon[89482]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:45.926 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:45.926 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:07:46.664 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:07:46.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:46 smithi090 ceph-mon[99575]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:46 smithi090 ceph-mon[95814]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:46.928 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:07:47.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:46 smithi114 ceph-mon[89482]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:48.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:48 smithi090 ceph-mon[99575]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:48.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:48 smithi090 ceph-mon[95814]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:48.952 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:07:48.952 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:07:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:48 smithi114 ceph-mon[89482]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:49.890 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:49 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/322642387' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:49.890 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:49 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/322642387' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:50.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:49 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/322642387' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:50.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:50 smithi090 ceph-mon[99575]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:50.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:50 smithi090 ceph-mon[95814]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:51.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:50 smithi114 ceph-mon[89482]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:52.627 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:07:52.899 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:07:52.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:52 smithi090 ceph-mon[99575]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:52.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:52 smithi090 ceph-mon[95814]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:53.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:52 smithi114 ceph-mon[89482]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:07:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:07:53] "GET /metrics HTTP/1.1" 200 34172 "" "Prometheus/2.43.0" 2024-04-12T09:07:54.844 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:07:54.845 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:07:54.919 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:54 smithi114 ceph-mon[89482]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:54.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:54 smithi090 ceph-mon[99575]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:54.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:54 smithi090 ceph-mon[95814]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:55.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:55 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/256171453' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:55.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:55 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/256171453' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:56.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:55 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/256171453' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:07:56.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:56 smithi090 ceph-mon[99575]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:56.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:56 smithi090 ceph-mon[95814]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:57.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:56 smithi114 ceph-mon[89482]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:58.606 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:07:58.880 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:07:58.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:07:58 smithi090 ceph-mon[99575]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:58.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:07:58 smithi090 ceph-mon[95814]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:07:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:07:58 smithi114 ceph-mon[89482]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:00.889 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:08:00.889 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:08:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:00 smithi114 ceph-mon[89482]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:01.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:00 smithi090 ceph-mon[99575]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:00 smithi090 ceph-mon[95814]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:01.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:01 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1472773108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:01.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:01 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1472773108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:02.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:01 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1472773108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:02 smithi114 ceph-mon[89482]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:03.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:02 smithi090 ceph-mon[99575]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:03.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:02 smithi090 ceph-mon[95814]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:03.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:08:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:08:03] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2024-04-12T09:08:04.647 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:08:04.917 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:08:05.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:04 smithi090 ceph-mon[95814]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:05.039 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:04 smithi090 ceph-mon[99575]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:04 smithi114 ceph-mon[89482]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:06.075 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:05 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:08:06.081 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:05 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:08:06.081 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:05 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:08:06.881 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:08:06.881 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:08:06.989 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:06 smithi090 ceph-mon[95814]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:06.990 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:06 smithi090 ceph-mon[99575]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:07.015 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:06 smithi114 ceph-mon[89482]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:07.980 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:07 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/356761016' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:08.172 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:07 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/356761016' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:08.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:07 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/356761016' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:08.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:08 smithi090 ceph-mon[99575]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:08.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:08 smithi090 ceph-mon[95814]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:09.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:08 smithi114 ceph-mon[89482]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:10.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:10 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:10.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:10 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:10.423 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:10 smithi090 ceph-mon[99575]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:10.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:10 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:10.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:10 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:10.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:10 smithi090 ceph-mon[95814]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:10.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:10 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:10.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:10 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:10.617 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:10 smithi114 ceph-mon[89482]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:10.731 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:08:10.991 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:08:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:11 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:11 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:11 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:08:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:11 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:08:11.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:11 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:11.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:11 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:11.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:11 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:11.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:08:11.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:08:11.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:11 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:11.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:11 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:11.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:11 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:11.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:08:11.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:08:11.924 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:11 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:08:12.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:12 smithi090 ceph-mon[95814]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:12.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:12 smithi090 ceph-mon[95814]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:12.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:12 smithi090 ceph-mon[95814]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:12.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:12 smithi090 ceph-mon[99575]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:12.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:12 smithi090 ceph-mon[99575]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:12.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:12 smithi090 ceph-mon[99575]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:12 smithi114 ceph-mon[89482]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:12 smithi114 ceph-mon[89482]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:12.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:12 smithi114 ceph-mon[89482]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:13.114 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:08:13.114 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:08:13.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:08:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:08:13] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2024-04-12T09:08:13.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:13 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1373729062' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:13.872 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:13 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1373729062' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:13.872 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:13 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1373729062' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:14.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:14 smithi114 ceph-mon[89482]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:14.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:14 smithi090 ceph-mon[99575]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:14 smithi090 ceph-mon[95814]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:15.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:15 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:15.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:15 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:15.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:16.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:16 smithi114 ceph-mon[89482]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:16.897 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:08:16.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:16 smithi090 ceph-mon[99575]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:16.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:16 smithi090 ceph-mon[95814]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:17.166 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:08:18.853 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:18 smithi090 ceph-mon[95814]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:18.853 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:18 smithi090 ceph-mon[99575]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:18.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:18 smithi114 ceph-mon[89482]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:19.358 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:08:19.358 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:08:19.736 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:19 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/4211819828' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:19.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:19 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/4211819828' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:20.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:19 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/4211819828' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:20.672 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:20 smithi090 ceph-mon[99575]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:20.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:20 smithi090 ceph-mon[95814]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:20 smithi114 ceph-mon[89482]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:22.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:22 smithi090 ceph-mon[99575]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:22.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:22 smithi090 ceph-mon[95814]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:23.113 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:08:23.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:22 smithi114 ceph-mon[89482]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:23.384 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:08:23.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:08:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:08:23] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.43.0" 2024-04-12T09:08:24.903 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:24 smithi090 ceph-mon[99575]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:24.903 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:24 smithi090 ceph-mon[95814]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:24.919 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:24 smithi114 ceph-mon[89482]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:25.399 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:08:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:08:25.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:25 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1017295407' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:25.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:25 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1017295407' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:25 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1017295407' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:26.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:26 smithi090 ceph-mon[95814]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:26.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:26 smithi090 ceph-mon[99575]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:27.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:26 smithi114 ceph-mon[89482]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:28.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:28 smithi090 ceph-mon[99575]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:28.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:28 smithi090 ceph-mon[95814]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:29.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:28 smithi114 ceph-mon[89482]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:29.118 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:08:29.387 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:08:30.725 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:30 smithi090 ceph-mon[99575]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:30.725 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:30 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:30.726 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:30 smithi090 ceph-mon[95814]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:30.726 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:30 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:30 smithi114 ceph-mon[89482]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:31.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:30 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:31.371 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:08:31.371 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:08:31.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:31 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1230407997' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:31.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:31 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1230407997' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:32.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:31 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1230407997' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:32.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:32 smithi090 ceph-mon[99575]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:32.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:32 smithi090 ceph-mon[95814]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:33.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:32 smithi114 ceph-mon[89482]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:33.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:08:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:08:33] "GET /metrics HTTP/1.1" 200 34169 "" "Prometheus/2.43.0" 2024-04-12T09:08:34.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:34 smithi090 ceph-mon[99575]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:34.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:34 smithi090 ceph-mon[95814]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:35.105 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:08:35.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:34 smithi114 ceph-mon[89482]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:35.375 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:08:36.846 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:36 smithi090 ceph-mon[95814]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:36.847 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:36 smithi090 ceph-mon[99575]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:37.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:36 smithi114 ceph-mon[89482]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:37.515 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:08:37.515 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:08:37.888 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:37 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1496261705' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:37.888 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:37 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1496261705' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:38.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:37 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1496261705' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:38.922 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:38 smithi090 ceph-mon[95814]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:38.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:38 smithi090 ceph-mon[99575]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:39.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:38 smithi114 ceph-mon[89482]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:40.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:40 smithi090 ceph-mon[95814]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:40.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:40 smithi090 ceph-mon[99575]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:41.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:40 smithi114 ceph-mon[89482]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:41.334 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:08:41.601 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:08:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:42 smithi090 ceph-mon[99575]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:42.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:42 smithi090 ceph-mon[95814]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:43.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:42 smithi114 ceph-mon[89482]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:43.322 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:08:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:08:43] "GET /metrics HTTP/1.1" 200 34169 "" "Prometheus/2.43.0" 2024-04-12T09:08:43.667 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:08:43.668 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:08:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:44 smithi090 ceph-mon[95814]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:44.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:44 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2599636774' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:44 smithi090 ceph-mon[99575]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:44.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:44 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2599636774' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:45.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:44 smithi114 ceph-mon[89482]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:45.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:44 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2599636774' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:45.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:45 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:45.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:45 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:46.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:45 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:08:46.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:46 smithi090 ceph-mon[99575]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:46.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:46 smithi090 ceph-mon[95814]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:47.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:46 smithi114 ceph-mon[89482]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:47.471 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:08:47.738 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:08:48.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:48 smithi090 ceph-mon[95814]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:48.934 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:48 smithi090 ceph-mon[99575]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:49.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:48 smithi114 ceph-mon[89482]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:50.049 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:08:50.049 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:08:51.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:50 smithi114 ceph-mon[89482]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:51.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:50 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/776260536' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:51.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:50 smithi090 ceph-mon[95814]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:51.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:50 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/776260536' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:51.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:50 smithi090 ceph-mon[99575]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:51.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:50 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/776260536' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:53.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:52 smithi114 ceph-mon[89482]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:53.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:52 smithi090 ceph-mon[99575]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:53.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:52 smithi090 ceph-mon[95814]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:53.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:08:53 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:08:53] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.43.0" 2024-04-12T09:08:53.839 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:08:54.109 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:08:54.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:54 smithi090 ceph-mon[99575]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:54.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:54 smithi090 ceph-mon[95814]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:55.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:54 smithi114 ceph-mon[89482]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:56.089 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:08:56.090 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:08:56.743 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:56 smithi090 ceph-mon[99575]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:56.743 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:56 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/566061011' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:56.744 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:56 smithi090 ceph-mon[95814]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:56.744 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:56 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/566061011' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:57.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:56 smithi114 ceph-mon[89482]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:57.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:56 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/566061011' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:08:59.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:08:58 smithi114 ceph-mon[89482]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:59.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:08:58 smithi090 ceph-mon[99575]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:59.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:08:58 smithi090 ceph-mon[95814]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:08:59.771 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:09:00.040 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:09:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:00 smithi114 ceph-mon[89482]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:01.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:00 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:09:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:00 smithi090 ceph-mon[95814]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:01.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:00 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:09:01.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:00 smithi090 ceph-mon[99575]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:01.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:00 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:09:02.077 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:09:02.077 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:09:02.733 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:02 smithi090 ceph-mon[99575]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:02.734 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:02 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/527262635' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:02.734 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:02 smithi090 ceph-mon[95814]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:02.734 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:02 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/527262635' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:02 smithi114 ceph-mon[89482]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:03.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:02 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/527262635' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:03.672 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:09:03] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.43.0" 2024-04-12T09:09:05.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:04 smithi114 ceph-mon[89482]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:05.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:04 smithi090 ceph-mon[95814]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:05.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:04 smithi090 ceph-mon[99575]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:05.759 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:09:06.026 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:09:07.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:06 smithi114 ceph-mon[89482]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:07.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:06 smithi090 ceph-mon[99575]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:07.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:06 smithi090 ceph-mon[95814]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:09:08.065 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:09:09.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:08 smithi114 ceph-mon[89482]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:09.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:08 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1286567722' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:09.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:08 smithi090 ceph-mon[99575]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:09.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:08 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1286567722' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:09.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:08 smithi090 ceph-mon[95814]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:09.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:08 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1286567722' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:11.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:10 smithi114 ceph-mon[89482]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:11.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:10 smithi090 ceph-mon[99575]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:11.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:10 smithi090 ceph-mon[95814]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:11.834 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:09:12.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:11 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:09:12.118 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:09:12.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:11 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:09:12.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:11 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T09:09:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:12 smithi090 ceph-mon[95814]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:12 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:12 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:12 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:12.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:12 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:12.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:12 smithi090 ceph-mon[99575]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:12.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:12 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:12.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:12 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:12.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:12 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:12.924 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:12 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:13.044 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:12 smithi114 ceph-mon[89482]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:13.044 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:12 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:13.044 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:12 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:13.044 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:12 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:13.044 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:12 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:13.556 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:13 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:09:13] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.43.0" 2024-04-12T09:09:14.193 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:09:14.193 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:09:14.922 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:14 smithi090 ceph-mon[99575]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:14.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:14 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2053996828' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:14 smithi090 ceph-mon[95814]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:14.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:14 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2053996828' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:15.061 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:14 smithi114 ceph-mon[89482]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:15.061 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:14 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2053996828' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:16.051 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:15 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:09:16.051 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:15 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:09:16.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:15 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:09:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:16 smithi090 ceph-mon[99575]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:16 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:16.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:16 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:16.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:16 smithi090 ceph-mon[95814]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:16.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:16 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:16.925 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:16 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:16 smithi114 ceph-mon[89482]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:16 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:17.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:16 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:18.404 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:09:18.667 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:09:19.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[95814]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:19.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:19.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:19.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:09:19.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:09:19.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[95814]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:19.042 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[99575]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:19.042 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:19.042 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:19.042 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:09:19.042 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:09:19.042 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:18 smithi090 ceph-mon[99575]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:19.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:18 smithi114 ceph-mon[89482]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:19.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:18 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:19.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:18 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:19.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:18 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T09:09:19.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:18 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T09:09:19.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:18 smithi114 ceph-mon[89482]: from='mgr.34104 ' entity='mgr.y' 2024-04-12T09:09:20.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:19 smithi114 ceph-mon[89482]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:20.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:19 smithi114 ceph-mon[89482]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:20.133 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:19 smithi090 ceph-mon[99575]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:20.133 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:19 smithi090 ceph-mon[99575]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:20.133 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:19 smithi090 ceph-mon[95814]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:20.133 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:19 smithi090 ceph-mon[95814]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:20.622 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:09:20.622 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:09:21.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:20 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2117935669' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:21.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:20 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2117935669' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:21.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:20 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2117935669' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:22.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:21 smithi114 ceph-mon[89482]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:22.172 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:21 smithi090 ceph-mon[99575]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:22.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:21 smithi090 ceph-mon[95814]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:23.673 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:23 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:09:23] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2024-04-12T09:09:24.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:23 smithi114 ceph-mon[89482]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:24.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:23 smithi090 ceph-mon[99575]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:24.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:23 smithi090 ceph-mon[95814]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:24.421 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:09:24.692 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:09:26.077 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:25 smithi090 ceph-mon[99575]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:26.077 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:25 smithi090 ceph-mon[95814]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:26.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:25 smithi114 ceph-mon[89482]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:26.722 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:09:26.722 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:09:27.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:26 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/1455768555' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:27.041 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:26 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/1455768555' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:27.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:26 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/1455768555' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:28.110 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:27 smithi114 ceph-mon[89482]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:28.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:27 smithi090 ceph-mon[95814]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:28.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:27 smithi090 ceph-mon[99575]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:29.117 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:28 smithi114 ceph-mon[89482]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:29.174 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:28 smithi090 ceph-mon[99575]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:29.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:28 smithi090 ceph-mon[95814]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:30.497 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:09:30.766 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:09:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:31 smithi114 ceph-mon[89482]: pgmap v674: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:31.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:31 smithi114 ceph-mon[89482]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:09:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:31 smithi090 ceph-mon[95814]: pgmap v674: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:31.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:31 smithi090 ceph-mon[95814]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:09:31.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:31 smithi090 ceph-mon[99575]: pgmap v674: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:31.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:31 smithi090 ceph-mon[99575]: from='mgr.34104 172.21.15.90:0/4098859172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T09:09:32.795 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:09:32.795 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:09:33.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:33 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:09:33] "GET /metrics HTTP/1.1" 200 34167 "" "Prometheus/2.43.0" 2024-04-12T09:09:33.723 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:33 smithi090 ceph-mon[99575]: pgmap v675: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:33.723 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:33 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/848298620' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:33.723 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:33 smithi090 ceph-mon[95814]: pgmap v675: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:33.724 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:33 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/848298620' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:33.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:33 smithi114 ceph-mon[89482]: pgmap v675: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:33.871 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:33 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/848298620' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:35.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:35 smithi114 ceph-mon[89482]: pgmap v676: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:35.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:35 smithi090 ceph-mon[99575]: pgmap v676: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:35.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:35 smithi090 ceph-mon[95814]: pgmap v676: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:36.527 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T09:09:36.796 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T09:09:37.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:37 smithi114 ceph-mon[89482]: pgmap v677: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:37.923 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:37 smithi090 ceph-mon[95814]: pgmap v677: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:37.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:37 smithi090 ceph-mon[99575]: pgmap v677: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:39.025 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-12T09:09:39.025 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-12T09:09:39.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:39 smithi090 ceph-mon[99575]: pgmap v678: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:39.673 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:39 smithi090 ceph-mon[99575]: from='client.? 172.21.15.90:0/2400346943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:39.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:39 smithi090 ceph-mon[95814]: pgmap v678: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:39.673 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:39 smithi090 ceph-mon[95814]: from='client.? 172.21.15.90:0/2400346943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:39.800 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_dea717a37c7849ca8459c02fb965020aedcc0a9a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_dea717a37c7849ca8459c02fb965020aedcc0a9a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-12T09:09:40.041 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=9f2bdfc1e09d43c98ffe01fb7a7b0ac6 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_dea717a37c7849ca8459c02fb965020aedcc0a9a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_dea717a37c7849ca8459c02fb965020aedcc0a9a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-12T09:09:40.043 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-12T09:09:40.051 INFO:tasks.cephadm:Teardown begin 2024-04-12T09:09:40.052 DEBUG:teuthology.orchestra.run.smithi090:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T09:09:40.056 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:39 smithi114 ceph-mon[89482]: pgmap v678: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-12T09:09:40.056 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:39 smithi114 ceph-mon[89482]: from='client.? 172.21.15.90:0/2400346943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T09:09:40.089 DEBUG:teuthology.orchestra.run.smithi114:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T09:09:40.165 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-12T09:09:40.165 DEBUG:teuthology.orchestra.run.smithi090:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-12T09:09:40.182 DEBUG:teuthology.orchestra.run.smithi114:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-12T09:09:40.199 INFO:tasks.cephadm:Stopping all daemons... 2024-04-12T09:09:40.200 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-12T09:09:40.200 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.a 2024-04-12T09:09:40.423 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:40 smithi090 systemd[1]: Stopping Ceph mon.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:09:40.920 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a[95810]: 2024-04-12T09:09:40.534+0000 7f5ff1470700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:09:40.921 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a[95810]: 2024-04-12T09:09:40.534+0000 7f5ff1470700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-12T09:09:40.921 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:40 smithi090 podman[150381]: 2024-04-12 09:09:40.627233449 +0000 UTC m=+0.204445348 container died ad8458aaf57951c945065cab63904c3774082128802f354dbdd0631c3d74d44c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, ceph=True) 2024-04-12T09:09:41.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:41 smithi090 podman[150381]: 2024-04-12 09:09:41.027111516 +0000 UTC m=+0.604323413 container cleanup ad8458aaf57951c945065cab63904c3774082128802f354dbdd0631c3d74d44c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-12T09:09:41.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 12 09:09:41 smithi090 bash[150381]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-a 2024-04-12T09:09:41.451 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.a.service' 2024-04-12T09:09:41.499 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:09:41.499 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-12T09:09:41.499 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-12T09:09:41.500 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.c 2024-04-12T09:09:41.795 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:41 smithi090 systemd[1]: Stopping Ceph mon.c for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:09:41.795 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c[99571]: 2024-04-12T09:09:41.783+0000 7fc521b71700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:09:41.796 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c[99571]: 2024-04-12T09:09:41.783+0000 7fc521b71700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-12T09:09:42.173 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:41 smithi090 podman[150502]: 2024-04-12 09:09:41.989189285 +0000 UTC m=+0.312398493 container died 6bfdbaae0edc6056f130989a9a37f3cac45c28369a63ad9679b9977294efe125 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-12T09:09:42.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:09:09:41] ENGINE Bus STOPPING 2024-04-12T09:09:42.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:09:09:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T09:09:42.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:09:09:41] ENGINE Bus STOPPED 2024-04-12T09:09:42.173 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:09:09:41] ENGINE Bus STARTING 2024-04-12T09:09:42.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:09:09:41] ENGINE Serving on http://:::9283 2024-04-12T09:09:42.174 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:41 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: [12/Apr/2024:09:09:41] ENGINE Bus STARTED 2024-04-12T09:09:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:42 smithi090 podman[150502]: 2024-04-12 09:09:42.709916235 +0000 UTC m=+1.033125430 container cleanup 6bfdbaae0edc6056f130989a9a37f3cac45c28369a63ad9679b9977294efe125 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-12T09:09:42.923 INFO:journalctl@ceph.mon.c.smithi090.stdout:Apr 12 09:09:42 smithi090 bash[150502]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-c 2024-04-12T09:09:43.078 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.c.service' 2024-04-12T09:09:43.114 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:09:43.115 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-12T09:09:43.115 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-12T09:09:43.115 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.b 2024-04-12T09:09:43.436 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:43 smithi114 systemd[1]: Stopping Ceph mon.b for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:09:43.674 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:43 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: ::ffff:172.21.15.114 - - [12/Apr/2024:09:09:43] "GET /metrics HTTP/1.1" 200 34167 "" "Prometheus/2.43.0" 2024-04-12T09:09:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:43 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b[89478]: 2024-04-12T09:09:43.434+0000 7f091fbea700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:09:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:43 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b[89478]: 2024-04-12T09:09:43.434+0000 7f091fbea700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-12T09:09:43.867 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:43 smithi114 podman[102131]: 2024-04-12 09:09:43.649213546 +0000 UTC m=+0.337180486 container died cd2de45bf1bf54bbe8467638c6a2d9ada8d431328156bed7f5547d646baa82c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-12T09:09:44.329 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:44 smithi114 podman[102131]: 2024-04-12 09:09:44.060766461 +0000 UTC m=+0.748733442 container cleanup cd2de45bf1bf54bbe8467638c6a2d9ada8d431328156bed7f5547d646baa82c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.build-date=20240409, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T09:09:44.329 INFO:journalctl@ceph.mon.b.smithi114.stdout:Apr 12 09:09:44 smithi114 bash[102131]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mon-b 2024-04-12T09:09:44.471 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mon.b.service' 2024-04-12T09:09:44.522 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:09:44.523 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-12T09:09:44.523 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-12T09:09:44.523 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.y 2024-04-12T09:09:44.833 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:44 smithi090 systemd[1]: Stopping Ceph mgr.y for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:09:45.152 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:09:44.832+0000 7efe622f7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:09:45.152 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y[86297]: 2024-04-12T09:09:44.832+0000 7efe622f7700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-12T09:09:45.152 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:44 smithi090 podman[150632]: 2024-04-12 09:09:44.864018975 +0000 UTC m=+0.149859545 container died b61a0f582c618169641e6300aedbf97a39978538f65549d8f235bf0b021da1de (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-12T09:09:45.422 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:45 smithi090 podman[150632]: 2024-04-12 09:09:45.334306411 +0000 UTC m=+0.620146966 container cleanup b61a0f582c618169641e6300aedbf97a39978538f65549d8f235bf0b021da1de (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-12T09:09:45.423 INFO:journalctl@ceph.mgr.y.smithi090.stdout:Apr 12 09:09:45 smithi090 bash[150632]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-y 2024-04-12T09:09:45.766 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.y.service' 2024-04-12T09:09:45.804 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:09:45.804 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-12T09:09:45.804 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-12T09:09:45.805 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.x 2024-04-12T09:09:46.117 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 09:09:45 smithi114 systemd[1]: Stopping Ceph mgr.x for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:09:46.417 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 09:09:46 smithi114 podman[102254]: 2024-04-12 09:09:46.13835349 +0000 UTC m=+0.144909070 container died 60c0ca408f9165df9aa655205a64edb0461d86d9a2d1f9e1daa87c85c71ae8f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T09:09:46.766 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 09:09:46 smithi114 podman[102254]: 2024-04-12 09:09:46.515874119 +0000 UTC m=+0.522429700 container cleanup 60c0ca408f9165df9aa655205a64edb0461d86d9a2d1f9e1daa87c85c71ae8f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb) 2024-04-12T09:09:46.767 INFO:journalctl@ceph.mgr.x.smithi114.stdout:Apr 12 09:09:46 smithi114 bash[102254]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-mgr-x 2024-04-12T09:09:46.939 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@mgr.x.service' 2024-04-12T09:09:46.975 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:09:46.975 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-12T09:09:46.976 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-12T09:09:46.976 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.0 2024-04-12T09:09:47.306 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:47 smithi090 systemd[1]: Stopping Ceph osd.0 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:09:47.672 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[108111]: 2024-04-12T09:09:47.305+0000 7f3e73291700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:09:47.673 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[108111]: 2024-04-12T09:09:47.305+0000 7f3e73291700 -1 osd.0 139 *** Got signal Terminated *** 2024-04-12T09:09:47.673 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:47 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0[108111]: 2024-04-12T09:09:47.305+0000 7f3e73291700 -1 osd.0 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T09:09:52.985 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:52 smithi090 podman[150754]: 2024-04-12 09:09:52.699356927 +0000 UTC m=+5.538686992 container died c245fffdfa5671d4b8e6ff5579dab5e815a9c1af8d4459a626c88d5ad8a2b94a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-12T09:09:53.423 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:53 smithi090 podman[150754]: 2024-04-12 09:09:53.192475751 +0000 UTC m=+6.031805824 container cleanup c245fffdfa5671d4b8e6ff5579dab5e815a9c1af8d4459a626c88d5ad8a2b94a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-12T09:09:53.423 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:53 smithi090 bash[150754]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0 2024-04-12T09:09:53.808 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:53 smithi090 podman[150771]: 2024-04-12 09:09:53.486337604 +0000 UTC m=+0.781522762 container remove c245fffdfa5671d4b8e6ff5579dab5e815a9c1af8d4459a626c88d5ad8a2b94a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-12T09:09:54.108 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:53 smithi090 podman[150863]: 2024-04-12 09:09:53.711442999 +0000 UTC m=+0.017570165 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:09:54.109 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:53 smithi090 podman[150863]: 2024-04-12 09:09:53.833450603 +0000 UTC m=+0.139577765 container create 75f7f124b0358dfb9b1c4e54a873911b2515c7d4e4c6bdc45b20df5467c7ffcc (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409) 2024-04-12T09:09:54.109 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:54 smithi090 podman[150863]: 2024-04-12 09:09:54.108728153 +0000 UTC m=+0.414855332 container init 75f7f124b0358dfb9b1c4e54a873911b2515c7d4e4c6bdc45b20df5467c7ffcc (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-12T09:09:54.378 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:09:54 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:09:54.305Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:09:54.378 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:09:54 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:09:54.306Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:09:54.378 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:54 smithi090 podman[150863]: 2024-04-12 09:09:54.112563709 +0000 UTC m=+0.418690879 container start 75f7f124b0358dfb9b1c4e54a873911b2515c7d4e4c6bdc45b20df5467c7ffcc (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:09:54.378 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:54 smithi090 podman[150863]: 2024-04-12 09:09:54.200351053 +0000 UTC m=+0.506478228 container attach 75f7f124b0358dfb9b1c4e54a873911b2515c7d4e4c6bdc45b20df5467c7ffcc (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T09:09:54.673 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 12 09:09:54 smithi090 podman[150863]: 2024-04-12 09:09:54.404100129 +0000 UTC m=+0.710227293 container died 75f7f124b0358dfb9b1c4e54a873911b2515c7d4e4c6bdc45b20df5467c7ffcc (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD) 2024-04-12T09:09:55.114 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.0.service' 2024-04-12T09:09:55.155 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:09:55.155 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-12T09:09:55.155 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-12T09:09:55.156 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.1 2024-04-12T09:09:55.517 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:09:55 smithi090 systemd[1]: Stopping Ceph osd.1 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:09:55.922 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:09:55 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[116777]: 2024-04-12T09:09:55.516+0000 7f6f8a08c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:09:55.923 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:09:55 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[116777]: 2024-04-12T09:09:55.516+0000 7f6f8a08c700 -1 osd.1 139 *** Got signal Terminated *** 2024-04-12T09:09:55.923 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:09:55 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1[116777]: 2024-04-12T09:09:55.516+0000 7f6f8a08c700 -1 osd.1 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T09:10:00.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:00.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:00.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:00 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:00.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:01.387 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:10:01 smithi090 podman[150983]: 2024-04-12 09:10:01.072112798 +0000 UTC m=+5.682238277 container died be4538ab3be1c3bb25d82b43a56ff08631a9e0587742a1312f60ba828f69cdf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-12T09:10:01.672 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:10:01 smithi090 podman[150983]: 2024-04-12 09:10:01.56970719 +0000 UTC m=+6.179832718 container cleanup be4538ab3be1c3bb25d82b43a56ff08631a9e0587742a1312f60ba828f69cdf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True) 2024-04-12T09:10:01.673 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:10:01 smithi090 bash[150983]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1 2024-04-12T09:10:02.142 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:10:01 smithi090 podman[151003]: 2024-04-12 09:10:01.838309556 +0000 UTC m=+0.761245014 container remove be4538ab3be1c3bb25d82b43a56ff08631a9e0587742a1312f60ba828f69cdf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:10:02.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:10:02 smithi090 podman[151093]: 2024-04-12 09:10:02.045205235 +0000 UTC m=+0.018291739 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:10:02.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:10:02 smithi090 podman[151093]: 2024-04-12 09:10:02.147727857 +0000 UTC m=+0.120814378 container create 79a2e2c3991be797589220654a741f2567f3cc98f1f8a869762816b23bd852e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-12T09:10:02.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:10:02 smithi090 podman[151093]: 2024-04-12 09:10:02.360861826 +0000 UTC m=+0.333948308 container init 79a2e2c3991be797589220654a741f2567f3cc98f1f8a869762816b23bd852e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb) 2024-04-12T09:10:02.423 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:10:02 smithi090 podman[151093]: 2024-04-12 09:10:02.364380212 +0000 UTC m=+0.337466689 container start 79a2e2c3991be797589220654a741f2567f3cc98f1f8a869762816b23bd852e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:10:02.923 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:10:02 smithi090 podman[151093]: 2024-04-12 09:10:02.468616741 +0000 UTC m=+0.441703222 container attach 79a2e2c3991be797589220654a741f2567f3cc98f1f8a869762816b23bd852e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:10:02.923 INFO:journalctl@ceph.osd.1.smithi090.stdout:Apr 12 09:10:02 smithi090 podman[151093]: 2024-04-12 09:10:02.659346224 +0000 UTC m=+0.632432705 container died 79a2e2c3991be797589220654a741f2567f3cc98f1f8a869762816b23bd852e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, RELEASE=squid-b277feb, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T09:10:03.284 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.1.service' 2024-04-12T09:10:03.328 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:10:03.329 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-12T09:10:03.329 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-12T09:10:03.329 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.2 2024-04-12T09:10:03.659 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:03 smithi090 systemd[1]: Stopping Ceph osd.2 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:10:03.922 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[125652]: 2024-04-12T09:10:03.658+0000 7f384046d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:10:03.923 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[125652]: 2024-04-12T09:10:03.658+0000 7f384046d700 -1 osd.2 139 *** Got signal Terminated *** 2024-04-12T09:10:03.923 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:03 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2[125652]: 2024-04-12T09:10:03.658+0000 7f384046d700 -1 osd.2 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T09:10:04.672 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:04 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:04.306Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:04.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:04 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:04.306Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:04.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:04 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:04.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:09.354 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:09 smithi090 podman[151198]: 2024-04-12 09:10:09.059808882 +0000 UTC m=+5.509986393 container died d798e05b6c642be094e0d9b14796199b3d58c7ffb1987a77b2745ff4cad5567a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-12T09:10:09.672 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:09 smithi090 podman[151198]: 2024-04-12 09:10:09.519902047 +0000 UTC m=+5.970079565 container cleanup d798e05b6c642be094e0d9b14796199b3d58c7ffb1987a77b2745ff4cad5567a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:10:09.673 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:09 smithi090 bash[151198]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2 2024-04-12T09:10:10.117 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:09.653Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.90:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.90:8765: connect: connection refused" 2024-04-12T09:10:10.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:09.655Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.90:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.90:8765: connect: connection refused" 2024-04-12T09:10:10.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:09.657Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.90:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.90:8765: connect: connection refused" 2024-04-12T09:10:10.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:09.657Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.90:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.90:8765: connect: connection refused" 2024-04-12T09:10:10.118 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:09 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:09.658Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.90:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.90:8765: connect: connection refused" 2024-04-12T09:10:10.138 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:09 smithi090 podman[151217]: 2024-04-12 09:10:09.797026583 +0000 UTC m=+0.739433417 container remove d798e05b6c642be094e0d9b14796199b3d58c7ffb1987a77b2745ff4cad5567a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux ) 2024-04-12T09:10:10.423 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:10 smithi090 podman[151307]: 2024-04-12 09:10:10.041240818 +0000 UTC m=+0.017978119 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:10:10.423 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:10 smithi090 podman[151307]: 2024-04-12 09:10:10.139560904 +0000 UTC m=+0.116298186 container create 2ca1a8eeb11ffe5e14ec88c6ae8d2ad7ddd80c50fd0f247f0b687b23dd955d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T09:10:10.423 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:10 smithi090 podman[151307]: 2024-04-12 09:10:10.376392306 +0000 UTC m=+0.353129589 container init 2ca1a8eeb11ffe5e14ec88c6ae8d2ad7ddd80c50fd0f247f0b687b23dd955d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-12T09:10:10.424 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:10 smithi090 podman[151307]: 2024-04-12 09:10:10.379636657 +0000 UTC m=+0.356373943 container start 2ca1a8eeb11ffe5e14ec88c6ae8d2ad7ddd80c50fd0f247f0b687b23dd955d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T09:10:10.424 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:10 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:10.160Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:10.424 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:10 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:10.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:10.424 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:10 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:10.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:10.923 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:10 smithi090 podman[151307]: 2024-04-12 09:10:10.477928286 +0000 UTC m=+0.454665581 container attach 2ca1a8eeb11ffe5e14ec88c6ae8d2ad7ddd80c50fd0f247f0b687b23dd955d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=) 2024-04-12T09:10:10.923 INFO:journalctl@ceph.osd.2.smithi090.stdout:Apr 12 09:10:10 smithi090 podman[151307]: 2024-04-12 09:10:10.673922998 +0000 UTC m=+0.650660281 container died 2ca1a8eeb11ffe5e14ec88c6ae8d2ad7ddd80c50fd0f247f0b687b23dd955d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-2-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-12T09:10:11.248 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.2.service' 2024-04-12T09:10:11.290 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:10:11.290 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-12T09:10:11.291 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-12T09:10:11.293 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.3 2024-04-12T09:10:11.619 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:11 smithi090 systemd[1]: Stopping Ceph osd.3 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:10:11.922 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:11 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3[130669]: 2024-04-12T09:10:11.618+0000 7fb246685700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:10:11.923 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:11 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3[130669]: 2024-04-12T09:10:11.618+0000 7fb246685700 -1 osd.3 139 *** Got signal Terminated *** 2024-04-12T09:10:11.923 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:11 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3[130669]: 2024-04-12T09:10:11.618+0000 7fb246685700 -1 osd.3 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T09:10:14.117 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:13 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:13.714+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:14.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:14 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:14.306Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:14.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:14 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:14.306Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:14.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:14 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:14.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:15.117 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:14 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:14.715+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:15.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:14 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:14.835+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:16.117 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:15.749+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:16.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:15 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:15.802+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:16.617 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:16 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:16.224+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:16.922 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:16 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3[130669]: 2024-04-12T09:10:16.516+0000 7fb23ec71700 -1 osd.3 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:50.239513+0000 front 2024-04-12T09:09:50.239387+0000 (oldest deadline 2024-04-12T09:10:16.139519+0000) 2024-04-12T09:10:17.117 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:16 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:16.707+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:17.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:16 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:16.813+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:17.422 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:17 smithi090 podman[151412]: 2024-04-12 09:10:17.168473068 +0000 UTC m=+5.655779658 container died d6cf9c940a8b59f07e3a0107ec16f0e210d271264d583a083b91b49d4742b3ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-12T09:10:17.617 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:17 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:17.207+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:17.880 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:17 smithi090 podman[151412]: 2024-04-12 09:10:17.629152002 +0000 UTC m=+6.116458624 container cleanup d6cf9c940a8b59f07e3a0107ec16f0e210d271264d583a083b91b49d4742b3ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-12T09:10:17.881 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:17 smithi090 bash[151412]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3 2024-04-12T09:10:18.117 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:17 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:17.679+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:18.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:17 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:17.771+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:18.172 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:17 smithi090 podman[151429]: 2024-04-12 09:10:17.880559853 +0000 UTC m=+0.714071657 container remove d6cf9c940a8b59f07e3a0107ec16f0e210d271264d583a083b91b49d4742b3ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-12T09:10:18.476 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:18 smithi090 podman[151520]: 2024-04-12 09:10:18.198312162 +0000 UTC m=+0.111458950 container create 72d57809d960d6e797a439a6fcf4620e0d1d1271ee9bbe7227af5b58a3ca9447 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-12T09:10:18.476 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:18 smithi090 podman[151520]: 2024-04-12 09:10:18.105082117 +0000 UTC m=+0.018228920 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T09:10:18.477 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:18 smithi090 podman[151520]: 2024-04-12 09:10:18.392765675 +0000 UTC m=+0.305912465 container init 72d57809d960d6e797a439a6fcf4620e0d1d1271ee9bbe7227af5b58a3ca9447 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-12T09:10:18.477 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:18 smithi090 podman[151520]: 2024-04-12 09:10:18.395776855 +0000 UTC m=+0.308923652 container start 72d57809d960d6e797a439a6fcf4620e0d1d1271ee9bbe7227af5b58a3ca9447 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-12T09:10:18.477 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:18 smithi090 podman[151520]: 2024-04-12 09:10:18.476261954 +0000 UTC m=+0.389408762 container attach 72d57809d960d6e797a439a6fcf4620e0d1d1271ee9bbe7227af5b58a3ca9447 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-12T09:10:18.578 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:18.164+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:18.867 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:18.635+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:18.867 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:18.773+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:18.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:18 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:18.578+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:18.923 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 12 09:10:18 smithi090 podman[151520]: 2024-04-12 09:10:18.686934563 +0000 UTC m=+0.600081343 container died 72d57809d960d6e797a439a6fcf4620e0d1d1271ee9bbe7227af5b58a3ca9447 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240409, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-12T09:10:19.370 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.3.service' 2024-04-12T09:10:19.411 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:10:19.412 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-12T09:10:19.412 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-12T09:10:19.412 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.4 2024-04-12T09:10:19.443 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:19.148+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:19.719 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:19.555+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:19.719 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:19 smithi114 systemd[1]: Stopping Ceph osd.4 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:10:19.719 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:19.591+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:19.719 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:19.717+0000 7f190c4a2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:10:19.719 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:19.717+0000 7f190c4a2700 -1 osd.4 139 *** Got signal Terminated *** 2024-04-12T09:10:20.117 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:19.717+0000 7f190c4a2700 -1 osd.4 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T09:10:20.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:19 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:19.738+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:20.422 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:20.161Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:20.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:20.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:20.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:20 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:20.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:20.559 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:20.162+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:20.867 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:20.746+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:20.867 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:20.624+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:20.868 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:20 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:20.557+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:21.558 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:21.120+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:21.867 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:21.771+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:21.867 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:21.658+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:21.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:21.558+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:21.868 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:21 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:21.558+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:22.597 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:22 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:22.149+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:22.598 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:22 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:22.149+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:22.867 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:22 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:22.815+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:22.867 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:22 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:22.628+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:22.868 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:22 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:22.596+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:22.868 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:22 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:22.596+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:23.555 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:23.190+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:23.556 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:23.190+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:23.831 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:23.554+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:23.831 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:23.554+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:23.832 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:23.659+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:23.832 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:23.659+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:10:00.111198+0000 front 2024-04-12T09:10:00.111192+0000 (oldest deadline 2024-04-12T09:10:23.611131+0000) 2024-04-12T09:10:24.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:23 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:23.830+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:24.549 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:24.198+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:24.549 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:24.198+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:24.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:24.307Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:24.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:24.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:24.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:24 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:24.308Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:24.861 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:24.548+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:24.862 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:24.548+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:24.862 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:24.676+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:47.810112+0000 front 2024-04-12T09:09:47.810097+0000 (oldest deadline 2024-04-12T09:10:13.109761+0000) 2024-04-12T09:10:24.862 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4[38377]: 2024-04-12T09:10:24.676+0000 7f190a2a3700 -1 osd.4 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:10:00.111198+0000 front 2024-04-12T09:10:00.111192+0000 (oldest deadline 2024-04-12T09:10:23.611131+0000) 2024-04-12T09:10:25.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:24.859+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:25.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:24 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:24.859+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.751057+0000 front 2024-04-12T09:09:58.751069+0000 (oldest deadline 2024-04-12T09:10:24.650946+0000) 2024-04-12T09:10:25.595 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:25.179+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:25.596 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:25.179+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:25.596 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:25 smithi114 podman[102396]: 2024-04-12 09:10:25.360014656 +0000 UTC m=+5.755578683 container died 15d0d85633921f0bf1ffc0948ebc60059d0b9f70e6c297b4828fdc2621e3a8d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-d3b5060, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True) 2024-04-12T09:10:25.862 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:25 smithi114 podman[102396]: 2024-04-12 09:10:25.693378018 +0000 UTC m=+6.088942087 container cleanup 15d0d85633921f0bf1ffc0948ebc60059d0b9f70e6c297b4828fdc2621e3a8d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4, org.label-schema.vendor=CentOS, RELEASE=quincy-d3b5060, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T09:10:25.862 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:25 smithi114 bash[102396]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4 2024-04-12T09:10:25.862 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:25.594+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:25.862 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:25.594+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:26.117 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:25 smithi114 podman[102416]: 2024-04-12 09:10:25.869459972 +0000 UTC m=+0.512661556 container remove 15d0d85633921f0bf1ffc0948ebc60059d0b9f70e6c297b4828fdc2621e3a8d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-12T09:10:26.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:25.860+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:26.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:25 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:25.860+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.751057+0000 front 2024-04-12T09:09:58.751069+0000 (oldest deadline 2024-04-12T09:10:24.650946+0000) 2024-04-12T09:10:26.499 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:26 smithi114 podman[102510]: 2024-04-12 09:10:26.295653177 +0000 UTC m=+0.112619446 container create 21454bc53bdc0a907e88fe1942ed4f7db5ad3f50bba2cc73a6ba20016c0cb85f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, RELEASE=quincy-d3b5060, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:10:26.499 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:26 smithi114 podman[102510]: 2024-04-12 09:10:26.201468385 +0000 UTC m=+0.018434600 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-12T09:10:26.500 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:26 smithi114 podman[102510]: 2024-04-12 09:10:26.498648598 +0000 UTC m=+0.315614799 container init 21454bc53bdc0a907e88fe1942ed4f7db5ad3f50bba2cc73a6ba20016c0cb85f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-12T09:10:26.500 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:26.199+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:26.500 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:26.199+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:26.853 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:26.614+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:26.853 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:26.614+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:26.853 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:26 smithi114 podman[102510]: 2024-04-12 09:10:26.502128784 +0000 UTC m=+0.319094987 container start 21454bc53bdc0a907e88fe1942ed4f7db5ad3f50bba2cc73a6ba20016c0cb85f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-12T09:10:26.853 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:26 smithi114 podman[102510]: 2024-04-12 09:10:26.623742466 +0000 UTC m=+0.440708679 container attach 21454bc53bdc0a907e88fe1942ed4f7db5ad3f50bba2cc73a6ba20016c0cb85f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=quincy-d3b5060, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-04-12T09:10:27.117 INFO:journalctl@ceph.osd.4.smithi114.stdout:Apr 12 09:10:26 smithi114 podman[102510]: 2024-04-12 09:10:26.853050259 +0000 UTC m=+0.670016468 container died 21454bc53bdc0a907e88fe1942ed4f7db5ad3f50bba2cc73a6ba20016c0cb85f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD) 2024-04-12T09:10:27.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:26.895+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:27.118 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:26 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:26.895+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.751057+0000 front 2024-04-12T09:09:58.751069+0000 (oldest deadline 2024-04-12T09:10:24.650946+0000) 2024-04-12T09:10:27.419 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:27.153+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:27.419 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:27.153+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:27.446 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.4.service' 2024-04-12T09:10:27.528 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:10:27.528 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-12T09:10:27.529 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-12T09:10:27.529 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.5 2024-04-12T09:10:27.705 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:27 smithi114 systemd[1]: Stopping Ceph osd.5 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:10:27.705 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:27.626+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:27.706 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:27.627+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:28.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:27.870+0000 7f5d15e24700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:10:28.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:27.870+0000 7f5d15e24700 -1 osd.5 139 *** Got signal Terminated *** 2024-04-12T09:10:28.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:27.870+0000 7f5d15e24700 -1 osd.5 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T09:10:28.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:27.874+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:28.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:27 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:27.874+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.751057+0000 front 2024-04-12T09:09:58.751069+0000 (oldest deadline 2024-04-12T09:10:24.650946+0000) 2024-04-12T09:10:28.617 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:28.151+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:28.617 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:28.151+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:28.909 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:28.641+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:28.910 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:28.641+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:29.367 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:28.908+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:29.367 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:28 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:28.908+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.751057+0000 front 2024-04-12T09:09:58.751069+0000 (oldest deadline 2024-04-12T09:10:24.650946+0000) 2024-04-12T09:10:29.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:29 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:29.141+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:29.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:29 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:29.141+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:29.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:29 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:29.612+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:29.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:29 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:29.612+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:30.367 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:29 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:29.944+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:30.367 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:29 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:29.944+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.751057+0000 front 2024-04-12T09:09:58.751069+0000 (oldest deadline 2024-04-12T09:10:24.650946+0000) 2024-04-12T09:10:30.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:30 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:30.167+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:30.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:30 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:30.167+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:30.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:30.161Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:30.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:30.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:30.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:30 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:30.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:30.930 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:30 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:30.626+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:30.930 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:30 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:30.626+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:31.367 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:30 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:30.929+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:31.367 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:30 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:30.929+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.751057+0000 front 2024-04-12T09:09:58.751069+0000 (oldest deadline 2024-04-12T09:10:24.650946+0000) 2024-04-12T09:10:31.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:31 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:31.135+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:31.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:31 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:31.135+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:31.949 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:31 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:31.662+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:31.949 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:31 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:31.662+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:31.949 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:31 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:31.662+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:05.490687+0000 front 2024-04-12T09:10:05.490752+0000 (oldest deadline 2024-04-12T09:10:31.291069+0000) 2024-04-12T09:10:32.367 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:31 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:31.948+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:48.150108+0000 front 2024-04-12T09:09:48.150191+0000 (oldest deadline 2024-04-12T09:10:14.050017+0000) 2024-04-12T09:10:32.367 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:31 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:31.948+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.751057+0000 front 2024-04-12T09:09:58.751069+0000 (oldest deadline 2024-04-12T09:10:24.650946+0000) 2024-04-12T09:10:32.368 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:31 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5[44013]: 2024-04-12T09:10:31.948+0000 7f5d0f41c700 -1 osd.5 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:06.951886+0000 front 2024-04-12T09:10:06.951950+0000 (oldest deadline 2024-04-12T09:10:31.651760+0000) 2024-04-12T09:10:32.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:32 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:32.165+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:32.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:32 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:32.165+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:32.369 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:32 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:32.165+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:33.117 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:32 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:32.618+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:33.117 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:32 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:32.618+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:33.117 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:32 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:32.618+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:05.490687+0000 front 2024-04-12T09:10:05.490752+0000 (oldest deadline 2024-04-12T09:10:31.291069+0000) 2024-04-12T09:10:33.567 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:33 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:33.144+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:33.568 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:33 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:33.144+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:33.568 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:33 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:33.144+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:33.867 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:33 smithi114 podman[102615]: 2024-04-12 09:10:33.566910462 +0000 UTC m=+5.809968954 container died 54049e134882bcc40378589a113ffe3b2d9fde85ba84cce2bf88444e64ed4831 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5, ceph=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-d3b5060, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-12T09:10:33.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:33 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:33.575+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:33.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:33 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:33.575+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:33.868 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:33 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:33.575+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:05.490687+0000 front 2024-04-12T09:10:05.490752+0000 (oldest deadline 2024-04-12T09:10:31.291069+0000) 2024-04-12T09:10:34.215 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:33 smithi114 podman[102615]: 2024-04-12 09:10:33.964703907 +0000 UTC m=+6.207762398 container cleanup 54049e134882bcc40378589a113ffe3b2d9fde85ba84cce2bf88444e64ed4831 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-12T09:10:34.216 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:33 smithi114 bash[102615]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5 2024-04-12T09:10:34.216 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:34 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:34.107+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:34.216 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:34 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:34.107+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:34.216 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:34 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:34.108+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:34.564 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:34 smithi114 podman[102645]: 2024-04-12 09:10:34.215844124 +0000 UTC m=+0.652707765 container remove 54049e134882bcc40378589a113ffe3b2d9fde85ba84cce2bf88444e64ed4831 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T09:10:34.672 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:34 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:34.308Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:34.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:34 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:34.308Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:34.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:34 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:34.308Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:34.861 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:34 smithi114 podman[102737]: 2024-04-12 09:10:34.468226392 +0000 UTC m=+0.018523764 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-12T09:10:34.861 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:34 smithi114 podman[102737]: 2024-04-12 09:10:34.575174531 +0000 UTC m=+0.125471944 container create 3a5a3326237d78d964533459bae72d98b143bc122310699ffb4559f958da0a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS) 2024-04-12T09:10:34.861 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:34 smithi114 podman[102737]: 2024-04-12 09:10:34.786065547 +0000 UTC m=+0.336362918 container init 3a5a3326237d78d964533459bae72d98b143bc122310699ffb4559f958da0a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, GIT_CLEAN=True) 2024-04-12T09:10:34.861 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:34 smithi114 podman[102737]: 2024-04-12 09:10:34.789172536 +0000 UTC m=+0.339469914 container start 3a5a3326237d78d964533459bae72d98b143bc122310699ffb4559f958da0a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-12T09:10:34.862 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:34 smithi114 podman[102737]: 2024-04-12 09:10:34.860864886 +0000 UTC m=+0.411162273 container attach 3a5a3326237d78d964533459bae72d98b143bc122310699ffb4559f958da0a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5-deactivate, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-12T09:10:34.862 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:34 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:34.562+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:34.862 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:34 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:34.563+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:34.862 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:34 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:34.563+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:05.490687+0000 front 2024-04-12T09:10:05.490752+0000 (oldest deadline 2024-04-12T09:10:31.291069+0000) 2024-04-12T09:10:35.117 INFO:journalctl@ceph.osd.5.smithi114.stdout:Apr 12 09:10:35 smithi114 podman[102737]: 2024-04-12 09:10:35.072166666 +0000 UTC m=+0.622464029 container died 3a5a3326237d78d964533459bae72d98b143bc122310699ffb4559f958da0a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-12T09:10:35.592 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:35.119+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:35.592 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:35.119+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:35.592 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:35.119+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:35.655 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.5.service' 2024-04-12T09:10:35.696 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:10:35.696 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-12T09:10:35.696 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-12T09:10:35.697 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.6 2024-04-12T09:10:35.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:35.590+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:35.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:35.590+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:35.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:35 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:35.590+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:05.490687+0000 front 2024-04-12T09:10:05.490752+0000 (oldest deadline 2024-04-12T09:10:31.291069+0000) 2024-04-12T09:10:35.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:35 smithi114 systemd[1]: Stopping Ceph osd.6 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:10:36.367 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:36 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:36.033+0000 7f2cb1caa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:10:36.367 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:36 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:36.033+0000 7f2cb1caa700 -1 osd.6 139 *** Got signal Terminated *** 2024-04-12T09:10:36.367 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:36 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:36.033+0000 7f2cb1caa700 -1 osd.6 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T09:10:36.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:36 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:36.123+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:36.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:36 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:36.123+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:36.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:36 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:36.123+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:36.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:36 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:36.614+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:36.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:36 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:36.614+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:36.868 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:36 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:36.614+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:05.490687+0000 front 2024-04-12T09:10:05.490752+0000 (oldest deadline 2024-04-12T09:10:31.291069+0000) 2024-04-12T09:10:37.589 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:37 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:37.150+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:37.589 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:37 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:37.150+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:37.589 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:37 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:37.150+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:37.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:37 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:37.588+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:37.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:37 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:37.588+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:37.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:37 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:37.588+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:05.490687+0000 front 2024-04-12T09:10:05.490752+0000 (oldest deadline 2024-04-12T09:10:31.291069+0000) 2024-04-12T09:10:38.587 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:38.126+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:38.587 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:38.126+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:38.588 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:38.126+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:38.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:38.587+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:38.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:38.587+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:38.868 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:38 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:38.587+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:05.490687+0000 front 2024-04-12T09:10:05.490752+0000 (oldest deadline 2024-04-12T09:10:31.291069+0000) 2024-04-12T09:10:39.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:39.093+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:39.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:39.093+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:39.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:39.093+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:39.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:39.564+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:39.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:39.564+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:39.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:39.564+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:05.490687+0000 front 2024-04-12T09:10:05.490752+0000 (oldest deadline 2024-04-12T09:10:31.291069+0000) 2024-04-12T09:10:39.868 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:39 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:39.564+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:15.392193+0000 front 2024-04-12T09:10:15.392149+0000 (oldest deadline 2024-04-12T09:10:38.892110+0000) 2024-04-12T09:10:40.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:40.163Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:40.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:40.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:40.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:40 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:40.164Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:40.571 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:40 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:40.117+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:40.571 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:40 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:40.117+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:40.571 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:40 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:40.117+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:40.571 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:40 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:40.117+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:16.181611+0000 front 2024-04-12T09:10:16.181566+0000 (oldest deadline 2024-04-12T09:10:39.681436+0000) 2024-04-12T09:10:40.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:40 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:40.570+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.088571+0000 front 2024-04-12T09:09:52.088602+0000 (oldest deadline 2024-04-12T09:10:18.488987+0000) 2024-04-12T09:10:40.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:40 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:40.570+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:58.489622+0000 front 2024-04-12T09:09:58.489721+0000 (oldest deadline 2024-04-12T09:10:21.389627+0000) 2024-04-12T09:10:40.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:40 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:40.570+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:05.490687+0000 front 2024-04-12T09:10:05.490752+0000 (oldest deadline 2024-04-12T09:10:31.291069+0000) 2024-04-12T09:10:40.867 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:40 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6[49730]: 2024-04-12T09:10:40.570+0000 7f2cafaab700 -1 osd.6 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:15.392193+0000 front 2024-04-12T09:10:15.392149+0000 (oldest deadline 2024-04-12T09:10:38.892110+0000) 2024-04-12T09:10:41.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:41 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:41.085+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:41.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:41 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:41.085+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:41.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:41 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:41.085+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:41.369 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:41 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:41.085+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:16.181611+0000 front 2024-04-12T09:10:16.181566+0000 (oldest deadline 2024-04-12T09:10:39.681436+0000) 2024-04-12T09:10:41.727 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:41 smithi114 podman[102841]: 2024-04-12 09:10:41.448908431 +0000 UTC m=+5.534070207 container died 86dfedc8629b58661e8fcde111d7fe9fd995fc6934e47f2dc1841993e0611efa (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=quincy-d3b5060, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:10:42.069 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:41 smithi114 podman[102841]: 2024-04-12 09:10:41.800902743 +0000 UTC m=+5.886064518 container cleanup 86dfedc8629b58661e8fcde111d7fe9fd995fc6934e47f2dc1841993e0611efa (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-d3b5060, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-12T09:10:42.069 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:41 smithi114 bash[102841]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6 2024-04-12T09:10:42.367 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:42 smithi114 podman[102861]: 2024-04-12 09:10:42.068810451 +0000 UTC m=+0.615523603 container remove 86dfedc8629b58661e8fcde111d7fe9fd995fc6934e47f2dc1841993e0611efa (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T09:10:42.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:42 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:42.093+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:42.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:42 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:42.093+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:42.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:42 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:42.093+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:42.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:42 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:42.093+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:16.181611+0000 front 2024-04-12T09:10:16.181566+0000 (oldest deadline 2024-04-12T09:10:39.681436+0000) 2024-04-12T09:10:42.739 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:42 smithi114 podman[102954]: 2024-04-12 09:10:42.312906036 +0000 UTC m=+0.018281257 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-12T09:10:42.739 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:42 smithi114 podman[102954]: 2024-04-12 09:10:42.440949373 +0000 UTC m=+0.146324625 container create ac49ba0ce173a80d4e6aa4d114a813a09b95fb2180034f1c65b49882221e0d9c (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6-deactivate, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-12T09:10:42.740 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:42 smithi114 podman[102954]: 2024-04-12 09:10:42.656191545 +0000 UTC m=+0.361566754 container init ac49ba0ce173a80d4e6aa4d114a813a09b95fb2180034f1c65b49882221e0d9c (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6-deactivate, RELEASE=quincy-d3b5060, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-12T09:10:42.740 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:42 smithi114 podman[102954]: 2024-04-12 09:10:42.65925855 +0000 UTC m=+0.364633760 container start ac49ba0ce173a80d4e6aa4d114a813a09b95fb2180034f1c65b49882221e0d9c (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-04-12T09:10:42.740 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:42 smithi114 podman[102954]: 2024-04-12 09:10:42.739339038 +0000 UTC m=+0.444714265 container attach ac49ba0ce173a80d4e6aa4d114a813a09b95fb2180034f1c65b49882221e0d9c (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux ) 2024-04-12T09:10:43.045 INFO:journalctl@ceph.osd.6.smithi114.stdout:Apr 12 09:10:42 smithi114 podman[102954]: 2024-04-12 09:10:42.93638217 +0000 UTC m=+0.641757380 container died ac49ba0ce173a80d4e6aa4d114a813a09b95fb2180034f1c65b49882221e0d9c (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-6-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-12T09:10:43.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:43 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:43.044+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:43.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:43 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:43.044+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:43.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:43 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:43.044+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:43.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:43 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:43.044+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:16.181611+0000 front 2024-04-12T09:10:16.181566+0000 (oldest deadline 2024-04-12T09:10:39.681436+0000) 2024-04-12T09:10:43.596 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.6.service' 2024-04-12T09:10:43.638 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:10:43.638 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-12T09:10:43.639 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-12T09:10:43.639 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.7 2024-04-12T09:10:43.867 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:43 smithi114 systemd[1]: Stopping Ceph osd.7 for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:10:44.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:43 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:43.978+0000 7f74b8e60700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T09:10:44.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:43 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:43.978+0000 7f74b8e60700 -1 osd.7 139 *** Got signal Terminated *** 2024-04-12T09:10:44.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:43 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:43.978+0000 7f74b8e60700 -1 osd.7 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T09:10:44.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:44 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:44.082+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:44.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:44 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:44.082+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:44.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:44 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:44.082+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:44.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:44 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:44.082+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:16.181611+0000 front 2024-04-12T09:10:16.181566+0000 (oldest deadline 2024-04-12T09:10:39.681436+0000) 2024-04-12T09:10:44.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:44.309Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:44.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:44.309Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:44.673 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:44 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:44.310Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:45.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:45 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:45.091+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:45.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:45 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:45.091+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:45.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:45 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:45.091+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:45.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:45 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:45.091+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:16.181611+0000 front 2024-04-12T09:10:16.181566+0000 (oldest deadline 2024-04-12T09:10:39.681436+0000) 2024-04-12T09:10:46.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:46 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:46.106+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:46.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:46 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:46.106+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:46.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:46 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:46.106+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:46.368 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:46 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:46.106+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:16.181611+0000 front 2024-04-12T09:10:16.181566+0000 (oldest deadline 2024-04-12T09:10:39.681436+0000) 2024-04-12T09:10:47.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:47 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:47.110+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:47.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:47 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:47.110+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:47.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:47 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:47.110+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:47.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:47 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:47.110+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:16.181611+0000 front 2024-04-12T09:10:16.181566+0000 (oldest deadline 2024-04-12T09:10:39.681436+0000) 2024-04-12T09:10:48.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:48.065+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-12T09:09:52.278341+0000 front 2024-04-12T09:09:52.278468+0000 (oldest deadline 2024-04-12T09:10:15.778182+0000) 2024-04-12T09:10:48.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:48.065+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6814 osd.1 since back 2024-04-12T09:09:59.279124+0000 front 2024-04-12T09:09:59.279233+0000 (oldest deadline 2024-04-12T09:10:21.578995+0000) 2024-04-12T09:10:48.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:48.065+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6822 osd.2 since back 2024-04-12T09:10:07.479896+0000 front 2024-04-12T09:10:07.479878+0000 (oldest deadline 2024-04-12T09:10:31.579858+0000) 2024-04-12T09:10:48.367 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:48 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7[55538]: 2024-04-12T09:10:48.065+0000 7f74b6c61700 -1 osd.7 139 heartbeat_check: no reply from 172.21.15.90:6830 osd.3 since back 2024-04-12T09:10:16.181611+0000 front 2024-04-12T09:10:16.181566+0000 (oldest deadline 2024-04-12T09:10:39.681436+0000) 2024-04-12T09:10:49.788 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:49 smithi114 podman[103060]: 2024-04-12 09:10:49.518006032 +0000 UTC m=+5.657975642 container died dda597b197122c8341439fda61cfa04e2a7d837f1667f25941b43c34f066977d (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-12T09:10:50.117 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:49 smithi114 podman[103060]: 2024-04-12 09:10:49.878821855 +0000 UTC m=+6.018791459 container cleanup dda597b197122c8341439fda61cfa04e2a7d837f1667f25941b43c34f066977d (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-12T09:10:50.117 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:49 smithi114 bash[103060]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7 2024-04-12T09:10:50.422 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:50 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:50.164Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:50.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:50 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:50.164Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:50.423 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:50 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:50.164Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:50.506 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:50 smithi114 podman[103077]: 2024-04-12 09:10:50.14677691 +0000 UTC m=+0.629528866 container remove dda597b197122c8341439fda61cfa04e2a7d837f1667f25941b43c34f066977d (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-12T09:10:50.783 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:50 smithi114 podman[103169]: 2024-04-12 09:10:50.409294894 +0000 UTC m=+0.017928925 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-12T09:10:50.783 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:50 smithi114 podman[103169]: 2024-04-12 09:10:50.522591867 +0000 UTC m=+0.131225898 container create 320fd4158b3001598b4378d2a09393da023fb51151b43e5b4b9a9399380adf2b (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-12T09:10:50.784 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:50 smithi114 podman[103169]: 2024-04-12 09:10:50.70874439 +0000 UTC m=+0.317378429 container init 320fd4158b3001598b4378d2a09393da023fb51151b43e5b4b9a9399380adf2b (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T09:10:50.784 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:50 smithi114 podman[103169]: 2024-04-12 09:10:50.712191054 +0000 UTC m=+0.320825094 container start 320fd4158b3001598b4378d2a09393da023fb51151b43e5b4b9a9399380adf2b (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-12T09:10:50.784 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:50 smithi114 podman[103169]: 2024-04-12 09:10:50.78336397 +0000 UTC m=+0.391998018 container attach 320fd4158b3001598b4378d2a09393da023fb51151b43e5b4b9a9399380adf2b (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=quincy-d3b5060) 2024-04-12T09:10:51.117 INFO:journalctl@ceph.osd.7.smithi114.stdout:Apr 12 09:10:50 smithi114 podman[103169]: 2024-04-12 09:10:50.986442161 +0000 UTC m=+0.595076187 container died 320fd4158b3001598b4378d2a09393da023fb51151b43e5b4b9a9399380adf2b (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-12T09:10:51.586 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@osd.7.service' 2024-04-12T09:10:51.627 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:10:51.627 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-12T09:10:51.627 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-12T09:10:51.627 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-96735376-f8a7-11ee-bc8c-c7b262605968@prometheus.a 2024-04-12T09:10:51.867 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 systemd[1]: Stopping Ceph prometheus.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:10:52.224 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.965Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-12T09:10:52.224 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.965Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-12T09:10:52.224 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.965Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-12T09:10:52.224 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.965Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-12T09:10:52.225 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.965Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-12T09:10:52.225 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.965Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-12T09:10:52.225 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.965Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-12T09:10:52.225 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.965Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-12T09:10:52.225 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.965Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-12T09:10:52.225 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.967Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-12T09:10:52.225 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.967Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-12T09:10:52.225 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a[79235]: ts=2024-04-12T09:10:51.967Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-12T09:10:52.225 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:51 smithi114 podman[103274]: 2024-04-12 09:10:51.976183507 +0000 UTC m=+0.132678144 container died f009118ca428198f620c5a61aa9bcdce3a3e72d7431061e66f0272b89a93ac8c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T09:10:52.617 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:52 smithi114 podman[103274]: 2024-04-12 09:10:52.323753837 +0000 UTC m=+0.480248475 container cleanup f009118ca428198f620c5a61aa9bcdce3a3e72d7431061e66f0272b89a93ac8c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T09:10:52.617 INFO:journalctl@ceph.prometheus.a.smithi114.stdout:Apr 12 09:10:52 smithi114 bash[103274]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-prometheus-a 2024-04-12T09:10:52.733 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-96735376-f8a7-11ee-bc8c-c7b262605968@prometheus.a.service' 2024-04-12T09:10:52.769 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T09:10:52.770 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-12T09:10:52.770 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 --force --keep-logs 2024-04-12T09:10:54.426 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:54 smithi090 systemd[1]: Stopping Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:10:54.427 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:54 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:54.310Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:54.427 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:54 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:54.310Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi090.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.90:8443: connect: connection refused" 2024-04-12T09:10:54.427 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:54 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:54.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-04-12T09:10:54.844 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:54 smithi090 ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a[85265]: ts=2024-04-12T09:10:54.513Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-12T09:10:54.844 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:54 smithi090 podman[151928]: 2024-04-12 09:10:54.55395303 +0000 UTC m=+0.144216180 container stop d11ba2dc752ae610824e54b7c3d94723bb8d055d163782c66b9f92559ea157f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T09:10:54.844 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:54 smithi090 podman[151928]: 2024-04-12 09:10:54.579493239 +0000 UTC m=+0.169756376 container died d11ba2dc752ae610824e54b7c3d94723bb8d055d163782c66b9f92559ea157f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T09:10:55.172 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:55 smithi090 podman[151928]: 2024-04-12 09:10:55.023429266 +0000 UTC m=+0.613692402 container cleanup d11ba2dc752ae610824e54b7c3d94723bb8d055d163782c66b9f92559ea157f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T09:10:55.173 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:55 smithi090 bash[151928]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a 2024-04-12T09:10:55.668 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:55 smithi090 podman[151943]: 2024-04-12 09:10:55.408759326 +0000 UTC m=+0.850029849 container remove d11ba2dc752ae610824e54b7c3d94723bb8d055d163782c66b9f92559ea157f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T09:10:55.669 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:55 smithi090 podman[151943]: 2024-04-12 09:10:55.495518791 +0000 UTC m=+0.936789261 volume remove 9828ec39c157dd62a36a107b677660ce700a1d3837386005240dfd01343651b8 2024-04-12T09:10:56.106 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 09:10:55 smithi090 systemd[1]: Stopping Ceph node-exporter.a for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:10:56.107 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:55 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-12T09:10:56.107 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:55 smithi090 systemd[1]: Stopped Ceph alertmanager.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T09:10:56.107 INFO:journalctl@ceph.alertmanager.a.smithi090.stdout:Apr 12 09:10:55 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@alertmanager.a.service: Consumed 2.277s CPU time. 2024-04-12T09:10:56.421 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 09:10:56 smithi090 podman[152048]: 2024-04-12 09:10:56.106091344 +0000 UTC m=+0.106465370 container died 687747c6880c9b137242934886c0c817b6c8e462738e3f58f0fbf0f2bf7881d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T09:10:56.672 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 09:10:56 smithi090 podman[152048]: 2024-04-12 09:10:56.529309138 +0000 UTC m=+0.529683153 container cleanup 687747c6880c9b137242934886c0c817b6c8e462738e3f58f0fbf0f2bf7881d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T09:10:56.673 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 09:10:56 smithi090 bash[152048]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a 2024-04-12T09:10:57.164 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 09:10:56 smithi090 podman[152062]: 2024-04-12 09:10:56.76568647 +0000 UTC m=+0.664224021 container remove 687747c6880c9b137242934886c0c817b6c8e462738e3f58f0fbf0f2bf7881d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T09:10:57.164 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 09:10:56 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-12T09:10:57.164 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 09:10:56 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-12T09:10:57.164 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 09:10:56 smithi090 systemd[1]: Stopped Ceph node-exporter.a for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T09:10:57.164 INFO:journalctl@ceph.node-exporter.a.smithi090.stdout:Apr 12 09:10:56 smithi090 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.a.service: Consumed 3.490s CPU time. 2024-04-12T09:11:06.385 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 --force --keep-logs 2024-04-12T09:11:08.117 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 09:11:07 smithi114 systemd[1]: Stopping Ceph node-exporter.b for 96735376-f8a7-11ee-bc8c-c7b262605968... 2024-04-12T09:11:08.569 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 09:11:08 smithi114 podman[103699]: 2024-04-12 09:11:08.246067889 +0000 UTC m=+0.108872466 container died e89762981b5565d3e57c7956c584040f732e7bad59524c54d27526b2284c879c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T09:11:08.569 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 09:11:08 smithi114 podman[103699]: 2024-04-12 09:11:08.568832853 +0000 UTC m=+0.431637445 container cleanup e89762981b5565d3e57c7956c584040f732e7bad59524c54d27526b2284c879c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T09:11:08.867 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 09:11:08 smithi114 bash[103699]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b 2024-04-12T09:11:08.867 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 09:11:08 smithi114 podman[103713]: 2024-04-12 09:11:08.769956148 +0000 UTC m=+0.526302962 container remove e89762981b5565d3e57c7956c584040f732e7bad59524c54d27526b2284c879c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-96735376-f8a7-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T09:11:08.867 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 09:11:08 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-12T09:11:09.329 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 09:11:08 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-12T09:11:09.329 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 09:11:08 smithi114 systemd[1]: Stopped Ceph node-exporter.b for 96735376-f8a7-11ee-bc8c-c7b262605968. 2024-04-12T09:11:09.330 INFO:journalctl@ceph.node-exporter.b.smithi114.stdout:Apr 12 09:11:08 smithi114 systemd[1]: ceph-96735376-f8a7-11ee-bc8c-c7b262605968@node-exporter.b.service: Consumed 3.361s CPU time. 2024-04-12T09:11:13.400 DEBUG:teuthology.orchestra.run.smithi090:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T09:11:13.436 DEBUG:teuthology.orchestra.run.smithi114:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T09:11:13.465 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-12T09:11:13.468 DEBUG:teuthology.misc:Transferring archived files from smithi090:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395/remote/smithi090/crash 2024-04-12T09:11:13.468 DEBUG:teuthology.orchestra.run.smithi090:> sudo tar c -f - -C /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/crash -- . 2024-04-12T09:11:13.531 INFO:teuthology.orchestra.run.smithi090.stderr:tar: /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/crash: Cannot open: No such file or directory 2024-04-12T09:11:13.531 INFO:teuthology.orchestra.run.smithi090.stderr:tar: Error is not recoverable: exiting now 2024-04-12T09:11:13.533 DEBUG:teuthology.misc:Transferring archived files from smithi114:/var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395/remote/smithi114/crash 2024-04-12T09:11:13.534 DEBUG:teuthology.orchestra.run.smithi114:> sudo tar c -f - -C /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/crash -- . 2024-04-12T09:11:13.576 INFO:teuthology.orchestra.run.smithi114.stderr:tar: /var/lib/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/crash: Cannot open: No such file or directory 2024-04-12T09:11:13.576 INFO:teuthology.orchestra.run.smithi114.stderr:tar: Error is not recoverable: exiting now 2024-04-12T09:11:13.577 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-12T09:11:13.578 DEBUG:teuthology.orchestra.run.smithi090:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-12T09:11:13.661 INFO:teuthology.orchestra.run.smithi090.stdout:1712911339.3388355 mon.a (mon.0) 693 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T09:11:13.661 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-12T09:11:13.661 INFO:tasks.cephadm:Compressing logs... 2024-04-12T09:11:13.661 DEBUG:teuthology.orchestra.run.smithi090:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-12T09:11:13.664 DEBUG:teuthology.orchestra.run.smithi114:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-12T09:11:13.692 INFO:teuthology.orchestra.run.smithi090.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-12T09:11:13.693 INFO:teuthology.orchestra.run.smithi090.stderr:: No such file or directory 2024-04-12T09:11:13.693 INFO:teuthology.orchestra.run.smithi090.stderr:gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mon.a.log 2024-04-12T09:11:13.693 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi090.rxdrqj.log 2024-04-12T09:11:13.693 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.audit.log 2024-04-12T09:11:13.693 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi090.rxdrqj.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.1.log 2024-04-12T09:11:13.693 INFO:teuthology.orchestra.run.smithi090.stderr: 82.0% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi090.rxdrqj.log.gz 2024-04-12T09:11:13.693 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.3.log 2024-04-12T09:11:13.694 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mgr.y.log 2024-04-12T09:11:13.694 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.0.log 2024-04-12T09:11:13.694 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.cephadm.log 2024-04-12T09:11:13.694 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.log 2024-04-12T09:11:13.694 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.2.log 2024-04-12T09:11:13.694 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-volume.log 2024-04-12T09:11:13.694 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mon.c.log 2024-04-12T09:11:13.698 INFO:teuthology.orchestra.run.smithi114.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-12T09:11:13.698 INFO:teuthology.orchestra.run.smithi114.stderr:: No such file or directory 2024-04-12T09:11:13.698 INFO:teuthology.orchestra.run.smithi114.stderr:gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.6.log 2024-04-12T09:11:13.698 INFO:teuthology.orchestra.run.smithi114.stderr:gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi114.nbzjob.log 2024-04-12T09:11:13.699 INFO:teuthology.orchestra.run.smithi114.stderr:gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.audit.log 2024-04-12T09:11:13.699 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.6.log: /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi114.nbzjob.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.4.log 2024-04-12T09:11:13.699 INFO:teuthology.orchestra.run.smithi090.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-volume.log: /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mon.c.log: 92.7% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.log.gz 2024-04-12T09:11:13.701 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/cephadm.log: 62.5% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi114.nbzjob.log.gz 2024-04-12T09:11:13.701 INFO:teuthology.orchestra.run.smithi114.stderr:gzip/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.audit.log: -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.7.log 2024-04-12T09:11:13.701 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.5.log 2024-04-12T09:11:13.702 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mgr.x.log 2024-04-12T09:11:13.702 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.cephadm.log 2024-04-12T09:11:13.702 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.log 2024-04-12T09:11:13.702 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-volume.log 2024-04-12T09:11:13.702 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mon.b.log 2024-04-12T09:11:13.791 INFO:teuthology.orchestra.run.smithi090.stderr: 90.6% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.cephadm.log.gz 2024-04-12T09:11:13.854 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-volume.log: /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mon.b.log: 84.1% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.cephadm.log.gz 2024-04-12T09:11:13.880 INFO:teuthology.orchestra.run.smithi114.stderr: 87.3% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.log.gz 2024-04-12T09:11:13.957 INFO:teuthology.orchestra.run.smithi114.stderr: 91.2% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-volume.log.gz 2024-04-12T09:11:13.966 INFO:teuthology.orchestra.run.smithi114.stderr: 89.4% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.audit.log.gz 2024-04-12T09:11:14.049 INFO:teuthology.orchestra.run.smithi114.stderr: 91.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-12T09:11:14.250 INFO:teuthology.orchestra.run.smithi090.stderr: 91.5% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mon.c.log.gz 2024-04-12T09:11:14.413 INFO:teuthology.orchestra.run.smithi090.stderr: 91.0% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-volume.log.gz 2024-04-12T09:11:14.500 INFO:teuthology.orchestra.run.smithi090.stderr: 93.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-12T09:11:14.501 INFO:teuthology.orchestra.run.smithi090.stderr: 93.6% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph.audit.log.gz 2024-04-12T09:11:14.824 INFO:teuthology.orchestra.run.smithi114.stderr: 89.6% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mgr.x.log.gz 2024-04-12T09:11:14.874 INFO:teuthology.orchestra.run.smithi090.stderr: 88.8% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mgr.y.log.gz 2024-04-12T09:11:17.558 INFO:teuthology.orchestra.run.smithi114.stderr: 92.0% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mon.b.log.gz 2024-04-12T09:11:18.483 INFO:teuthology.orchestra.run.smithi090.stderr: 90.4% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-mon.a.log.gz 2024-04-12T09:11:51.352 INFO:teuthology.orchestra.run.smithi114.stderr: 93.8% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.6.log.gz 2024-04-12T09:12:01.121 INFO:teuthology.orchestra.run.smithi114.stderr: 93.8% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.7.log.gz 2024-04-12T09:12:03.621 INFO:teuthology.orchestra.run.smithi114.stderr: 93.6% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.4.log.gz 2024-04-12T09:12:07.897 INFO:teuthology.orchestra.run.smithi090.stderr: 93.4% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.2.log.gz 2024-04-12T09:12:30.863 INFO:teuthology.orchestra.run.smithi114.stderr: 92.5% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.5.log.gz 2024-04-12T09:12:30.864 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-12T09:12:30.864 INFO:teuthology.orchestra.run.smithi114.stderr:real 1m17.186s 2024-04-12T09:12:30.864 INFO:teuthology.orchestra.run.smithi114.stderr:user 3m8.314s 2024-04-12T09:12:30.864 INFO:teuthology.orchestra.run.smithi114.stderr:sys 0m7.039s 2024-04-12T09:12:46.422 INFO:teuthology.orchestra.run.smithi090.stderr: 93.6% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.1.log.gz 2024-04-12T09:13:05.661 INFO:teuthology.orchestra.run.smithi090.stderr: 93.4% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.0.log.gz 2024-04-12T09:13:09.432 INFO:teuthology.orchestra.run.smithi090.stderr: 93.1% -- replaced with /var/log/ceph/96735376-f8a7-11ee-bc8c-c7b262605968/ceph-osd.3.log.gz 2024-04-12T09:13:09.434 INFO:teuthology.orchestra.run.smithi090.stderr: 2024-04-12T09:13:09.434 INFO:teuthology.orchestra.run.smithi090.stderr:real 1m55.758s 2024-04-12T09:13:09.435 INFO:teuthology.orchestra.run.smithi090.stderr:user 3m48.188s 2024-04-12T09:13:09.435 INFO:teuthology.orchestra.run.smithi090.stderr:sys 0m8.908s 2024-04-12T09:13:09.435 INFO:tasks.cephadm:Archiving logs... 2024-04-12T09:13:09.435 DEBUG:teuthology.misc:Transferring archived files from smithi090:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395/remote/smithi090/log 2024-04-12T09:13:09.436 DEBUG:teuthology.orchestra.run.smithi090:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-12T09:13:45.262 DEBUG:teuthology.misc:Transferring archived files from smithi114:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395/remote/smithi114/log 2024-04-12T09:13:45.264 DEBUG:teuthology.orchestra.run.smithi114:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-12T09:14:15.401 INFO:tasks.cephadm:Removing cluster... 2024-04-12T09:14:15.402 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 --force 2024-04-12T09:14:16.164 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 96735376-f8a7-11ee-bc8c-c7b262605968 --force 2024-04-12T09:14:16.916 INFO:tasks.cephadm:Removing cephadm ... 2024-04-12T09:14:16.916 DEBUG:teuthology.orchestra.run.smithi090:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-12T09:14:16.935 DEBUG:teuthology.orchestra.run.smithi114:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-12T09:14:16.949 INFO:tasks.cephadm:Teardown complete 2024-04-12T09:14:16.950 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-12T09:14:16.968 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-12T09:14:16.969 DEBUG:teuthology.orchestra.run.smithi090:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-12T09:14:16.978 DEBUG:teuthology.orchestra.run.smithi114:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-12T09:14:17.080 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-12T09:14:17.089 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-12T09:14:17.090 DEBUG:teuthology.orchestra.run.smithi090:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T09:14:17.092 DEBUG:teuthology.orchestra.run.smithi114:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T09:14:17.106 INFO:teuthology.orchestra.run.smithi090.stderr:bash: line 1: ntpq: command not found 2024-04-12T09:14:17.106 INFO:teuthology.orchestra.run.smithi114.stderr:bash: line 1: ntpq: command not found 2024-04-12T09:14:17.110 INFO:teuthology.orchestra.run.smithi114.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T09:14:17.111 INFO:teuthology.orchestra.run.smithi114.stdout:=============================================================================== 2024-04-12T09:14:17.111 INFO:teuthology.orchestra.run.smithi114.stdout:^* hv01.front.sepia.ceph.com 3 6 377 42 +534us[ +535us] +/- 64ms 2024-04-12T09:14:17.111 INFO:teuthology.orchestra.run.smithi114.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 39 -1143us[-1143us] +/- 58ms 2024-04-12T09:14:17.111 INFO:teuthology.orchestra.run.smithi114.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 44 +947us[ +948us] +/- 84ms 2024-04-12T09:14:17.111 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T09:14:17.112 INFO:teuthology.orchestra.run.smithi090.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T09:14:17.112 INFO:teuthology.orchestra.run.smithi090.stdout:=============================================================================== 2024-04-12T09:14:17.112 INFO:teuthology.orchestra.run.smithi090.stdout:^* hv01.front.sepia.ceph.com 3 6 377 42 +529us[ +538us] +/- 64ms 2024-04-12T09:14:17.113 INFO:teuthology.orchestra.run.smithi090.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 39 -1145us[-1145us] +/- 58ms 2024-04-12T09:14:17.113 INFO:teuthology.orchestra.run.smithi090.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 40 +953us[ +953us] +/- 84ms 2024-04-12T09:14:17.113 INFO:teuthology.orchestra.run.smithi090.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T09:14:17.113 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-12T09:14:17.123 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-12T09:14:17.124 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-12T09:14:17.144 DEBUG:teuthology.orchestra.run.smithi090:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-12T09:14:17.250 DEBUG:teuthology.orchestra.run.smithi114:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-12T09:14:17.423 DEBUG:teuthology.orchestra.run.smithi090:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-12T09:14:17.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T09:14:17.456 DEBUG:teuthology.orchestra.run.smithi114:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-12T09:14:17.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T09:14:17.487 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-12T09:14:17.497 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-12T09:14:17.519 INFO:teuthology.task.internal:Duration was 2792.298438 seconds 2024-04-12T09:14:17.519 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-12T09:14:17.553 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-12T09:14:17.553 DEBUG:teuthology.orchestra.run.smithi090:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-12T09:14:17.556 DEBUG:teuthology.orchestra.run.smithi114:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-12T09:14:17.598 INFO:teuthology.orchestra.run.smithi090.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T09:14:17.626 INFO:teuthology.orchestra.run.smithi114.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T09:14:17.960 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-12T09:14:17.961 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi090.front.sepia.ceph.com 2024-04-12T09:14:17.961 DEBUG:teuthology.orchestra.run.smithi090:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-12T09:14:17.983 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi114.front.sepia.ceph.com 2024-04-12T09:14:17.984 DEBUG:teuthology.orchestra.run.smithi114:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-12T09:14:18.013 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-12T09:14:18.013 DEBUG:teuthology.orchestra.run.smithi090:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-12T09:14:18.026 DEBUG:teuthology.orchestra.run.smithi114:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-12T09:14:18.082 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-12T09:14:18.082 DEBUG:teuthology.orchestra.run.smithi090:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-12T09:14:18.096 DEBUG:teuthology.orchestra.run.smithi114:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-12T09:14:18.557 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-12T09:14:18.569 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-12T09:14:18.569 DEBUG:teuthology.orchestra.run.smithi090:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-12T09:14:18.607 DEBUG:teuthology.orchestra.run.smithi114:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-12T09:14:18.635 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-12T09:14:18.648 DEBUG:teuthology.orchestra.run.smithi090:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-12T09:14:18.651 DEBUG:teuthology.orchestra.run.smithi114:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-12T09:14:18.674 INFO:teuthology.orchestra.run.smithi090.stdout:kernel.core_pattern = core 2024-04-12T09:14:18.706 INFO:teuthology.orchestra.run.smithi114.stdout:kernel.core_pattern = core 2024-04-12T09:14:18.731 DEBUG:teuthology.orchestra.run.smithi090:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-12T09:14:18.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T09:14:18.746 DEBUG:teuthology.orchestra.run.smithi114:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-12T09:14:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T09:14:18.786 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-12T09:14:18.800 INFO:teuthology.task.internal:Transferring archived files... 2024-04-12T09:14:18.801 DEBUG:teuthology.misc:Transferring archived files from smithi090:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395/remote/smithi090 2024-04-12T09:14:18.801 DEBUG:teuthology.orchestra.run.smithi090:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-12T09:14:18.852 DEBUG:teuthology.misc:Transferring archived files from smithi114:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395/remote/smithi114 2024-04-12T09:14:18.853 DEBUG:teuthology.orchestra.run.smithi114:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-12T09:14:18.893 INFO:teuthology.task.internal:Removing archive directory... 2024-04-12T09:14:18.893 DEBUG:teuthology.orchestra.run.smithi090:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-12T09:14:18.896 DEBUG:teuthology.orchestra.run.smithi114:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-12T09:14:18.941 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-12T09:14:18.954 INFO:teuthology.task.internal:Not uploading archives. 2024-04-12T09:14:18.954 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-12T09:14:18.980 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-12T09:14:18.980 DEBUG:teuthology.orchestra.run.smithi090:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-12T09:14:18.983 DEBUG:teuthology.orchestra.run.smithi114:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-12T09:14:18.995 INFO:teuthology.orchestra.run.smithi090.stdout: 266043 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 12 09:14 /home/ubuntu/cephtest 2024-04-12T09:14:18.996 INFO:teuthology.orchestra.run.smithi114.stdout: 265306 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 12 09:14 /home/ubuntu/cephtest 2024-04-12T09:14:18.998 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-12T09:14:19.016 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-12T09:14:19.081 INFO:teuthology.nuke:Checking targets against current locks 2024-04-12T09:14:19.108 DEBUG:teuthology.nuke:shortname: smithi090 2024-04-12T09:14:19.108 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T09:14:19.133 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi090.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 08:16:13.323032', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGqN0HyNQTnN1DzR4tPZQnLo35xJt4BjNPTkJg2QGWKxVXg8amz0AqCYugiw0t6WgiLxfK5Hl6VvRzOV09zbcVg='} 2024-04-12T09:14:19.136 DEBUG:teuthology.nuke:shortname: smithi114 2024-04-12T09:14:19.136 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T09:14:19.159 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi114.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_02:08:08-upgrade-squid-distro-default-smithi/7653395', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 08:16:13.321447', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPSxYvJSi9IeAMhdznrwLqJLH3yDDYmhIQ0mElMNI+MMohDyPYFjKRuNV8InIpNIW5G9dgzsbNhnH+ElEFebY4A='} 2024-04-12T09:14:19.190 INFO:teuthology.orchestra.console.smithi090:Power off 2024-04-12T09:14:19.191 DEBUG:teuthology.orchestra.console.smithi090:pexpect command: ipmitool -H smithi090.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-12T09:14:19.226 INFO:teuthology.orchestra.console.smithi114:Power off 2024-04-12T09:14:19.227 DEBUG:teuthology.orchestra.console.smithi114:pexpect command: ipmitool -H smithi114.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-12T09:14:19.235 DEBUG:teuthology.orchestra.console.smithi090:power off output: Chassis Power Control: Down/Off 2024-04-12T09:14:19.235 DEBUG:teuthology.orchestra.console.smithi090:pexpect command: ipmitool -H smithi090.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T09:14:19.251 DEBUG:teuthology.orchestra.console.smithi114:power off output: Chassis Power Control: Down/Off 2024-04-12T09:14:19.252 DEBUG:teuthology.orchestra.console.smithi114:pexpect command: ipmitool -H smithi114.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T09:14:19.261 DEBUG:teuthology.orchestra.console.smithi090:check power output: Chassis Power is on 2024-04-12T09:14:19.274 DEBUG:teuthology.orchestra.console.smithi114:check power output: Chassis Power is on 2024-04-12T09:14:23.262 DEBUG:teuthology.orchestra.console.smithi090:pexpect command: ipmitool -H smithi090.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T09:14:23.275 DEBUG:teuthology.orchestra.console.smithi114:pexpect command: ipmitool -H smithi114.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T09:14:23.373 DEBUG:teuthology.orchestra.console.smithi090:check power output: Chassis Power is on 2024-04-12T09:14:23.383 DEBUG:teuthology.orchestra.console.smithi114:check power output: Chassis Power is on 2024-04-12T09:14:27.379 DEBUG:teuthology.orchestra.console.smithi090:pexpect command: ipmitool -H smithi090.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T09:14:27.387 DEBUG:teuthology.orchestra.console.smithi114:pexpect command: ipmitool -H smithi114.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T09:14:27.490 DEBUG:teuthology.orchestra.console.smithi090:check power output: Chassis Power is off 2024-04-12T09:14:27.498 DEBUG:teuthology.orchestra.console.smithi114:check power output: Chassis Power is off 2024-04-12T09:14:27.592 INFO:teuthology.orchestra.console.smithi090:Power off completed 2024-04-12T09:14:27.599 INFO:teuthology.orchestra.console.smithi114:Power off completed 2024-04-12T09:14:27.704 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 2792.298437833786 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=9f2bdfc1e09d43c98ffe01fb7a7b0ac6 status: fail success: false 2024-04-12T09:14:27.704 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-12T09:14:27.780 INFO:teuthology.run:FAIL